mirror of
https://github.com/langchain-ai/langchain-benchmarks.git
synced 2026-07-20 00:53:46 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Eugene Yurtsev
|
b7796fce27 |
@@ -1,91 +0,0 @@
|
||||
# An action for setting up poetry install with caching.
|
||||
# Using a custom action since the default action does not
|
||||
# take poetry install groups into account.
|
||||
# Action code from:
|
||||
# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236
|
||||
name: poetry-install-with-caching
|
||||
description: Poetry install with support for caching of dependency groups.
|
||||
|
||||
inputs:
|
||||
python-version:
|
||||
description: Python version, supporting MAJOR.MINOR only
|
||||
required: true
|
||||
|
||||
poetry-version:
|
||||
description: Poetry version
|
||||
required: true
|
||||
|
||||
cache-key:
|
||||
description: Cache key to use for manual handling of caching
|
||||
required: true
|
||||
|
||||
working-directory:
|
||||
description: Directory whose poetry.lock file should be cached
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/setup-python@v4
|
||||
name: Setup python ${{ inputs.python-version }}
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
|
||||
- uses: actions/cache@v3
|
||||
id: cache-bin-poetry
|
||||
name: Cache Poetry binary - Python ${{ inputs.python-version }}
|
||||
env:
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
|
||||
with:
|
||||
path: |
|
||||
/opt/pipx/venvs/poetry
|
||||
# This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
|
||||
key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
|
||||
|
||||
- name: Refresh shell hashtable and fixup softlinks
|
||||
if: steps.cache-bin-poetry.outputs.cache-hit == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
POETRY_VERSION: ${{ inputs.poetry-version }}
|
||||
PYTHON_VERSION: ${{ inputs.python-version }}
|
||||
run: |
|
||||
set -eux
|
||||
|
||||
# Refresh the shell hashtable, to ensure correct `which` output.
|
||||
hash -r
|
||||
|
||||
# `actions/cache@v3` doesn't always seem able to correctly unpack softlinks.
|
||||
# Delete and recreate the softlinks pipx expects to have.
|
||||
rm /opt/pipx/venvs/poetry/bin/python
|
||||
cd /opt/pipx/venvs/poetry/bin
|
||||
ln -s "$(which "python$PYTHON_VERSION")" python
|
||||
chmod +x python
|
||||
cd /opt/pipx_bin/
|
||||
ln -s /opt/pipx/venvs/poetry/bin/poetry poetry
|
||||
chmod +x poetry
|
||||
|
||||
# Ensure everything got set up correctly.
|
||||
/opt/pipx/venvs/poetry/bin/python --version
|
||||
/opt/pipx_bin/poetry --version
|
||||
|
||||
- name: Install poetry
|
||||
if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
POETRY_VERSION: ${{ inputs.poetry-version }}
|
||||
PYTHON_VERSION: ${{ inputs.python-version }}
|
||||
run: pipx install "poetry==$POETRY_VERSION" --python "python$PYTHON_VERSION" --verbose
|
||||
|
||||
- name: Restore pip and poetry cached dependencies
|
||||
uses: actions/cache@v3
|
||||
env:
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
|
||||
WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
|
||||
with:
|
||||
path: |
|
||||
~/.cache/pip
|
||||
~/.cache/pypoetry/virtualenvs
|
||||
~/.cache/pypoetry/cache
|
||||
~/.cache/pypoetry/artifacts
|
||||
${{ env.WORKDIR }}/.venv
|
||||
key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}
|
||||
@@ -1,83 +0,0 @@
|
||||
name: lint
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
working-directory:
|
||||
required: true
|
||||
type: string
|
||||
description: "From which folder this pipeline executes"
|
||||
|
||||
env:
|
||||
POETRY_VERSION: "1.6.1"
|
||||
WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# This number is set "by eye": we want it to be big enough
|
||||
# so that it's bigger than the number of commits in any reasonable PR,
|
||||
# and also as small as possible since increasing the number makes
|
||||
# the initial `git fetch` slower.
|
||||
FETCH_DEPTH: 50
|
||||
strategy:
|
||||
matrix:
|
||||
# Only lint on the min and max supported Python versions.
|
||||
# It's extremely unlikely that there's a lint issue on any version in between
|
||||
# that doesn't show up on the min or max versions.
|
||||
#
|
||||
# GitHub rate-limits how many jobs can be running at any one time.
|
||||
# Starting new jobs is also relatively slow,
|
||||
# so linting on fewer versions makes CI faster.
|
||||
python-version:
|
||||
- "3.8"
|
||||
- "3.11"
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
cache-key: lint-with-extras
|
||||
|
||||
- name: Check Poetry File
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
poetry check
|
||||
|
||||
- name: Check lock file
|
||||
shell: bash
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
poetry lock --check
|
||||
|
||||
- name: Install dependencies
|
||||
# Also installs dev/lint/test/typing dependencies, to ensure we have
|
||||
# type hints for as many of our libraries as possible.
|
||||
# This helps catch errors that require dependencies to be spotted, for example:
|
||||
# https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
|
||||
#
|
||||
# If you change this configuration, make sure to change the `cache-key`
|
||||
# in the `poetry_setup` action above to stop using the old cache.
|
||||
# It doesn't matter how you change it, any change will cause a cache-bust.
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
poetry install --with dev,lint,test,typing
|
||||
|
||||
- name: Get .mypy_cache to speed up mypy
|
||||
uses: actions/cache@v3
|
||||
env:
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
|
||||
with:
|
||||
path: |
|
||||
${{ env.WORKDIR }}/.mypy_cache
|
||||
key: mypy-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }}
|
||||
|
||||
- name: Analysing the code with our lint
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
make lint
|
||||
@@ -1,94 +0,0 @@
|
||||
name: pydantic v1/v2 compatibility
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
working-directory:
|
||||
required: true
|
||||
type: string
|
||||
description: "From which folder this pipeline executes"
|
||||
|
||||
env:
|
||||
POETRY_VERSION: "1.6.1"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
timeout-minutes: 5
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.8"
|
||||
- "3.9"
|
||||
- "3.10"
|
||||
- "3.11"
|
||||
name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
cache-key: pydantic-cross-compat
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: poetry install --with test
|
||||
|
||||
- name: Install the opposite major version of pydantic
|
||||
# If normal tests use pydantic v1, here we'll use v2, and vice versa.
|
||||
shell: bash
|
||||
run: |
|
||||
# Determine the major part of pydantic version
|
||||
REGULAR_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
|
||||
|
||||
if [[ "$REGULAR_VERSION" == "1" ]]; then
|
||||
PYDANTIC_DEP=">=2.1,<3"
|
||||
TEST_WITH_VERSION="2"
|
||||
elif [[ "$REGULAR_VERSION" == "2" ]]; then
|
||||
PYDANTIC_DEP="<2"
|
||||
TEST_WITH_VERSION="1"
|
||||
else
|
||||
echo "Unexpected pydantic major version '$REGULAR_VERSION', cannot determine which version to use for cross-compatibility test."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install via `pip` instead of `poetry add` to avoid changing lockfile,
|
||||
# which would prevent caching from working: the cache would get saved
|
||||
# to a different key than where it gets loaded from.
|
||||
poetry run pip install "pydantic${PYDANTIC_DEP}"
|
||||
|
||||
# Ensure that the correct pydantic is installed now.
|
||||
echo "Checking pydantic version... Expecting ${TEST_WITH_VERSION}"
|
||||
|
||||
# Determine the major part of pydantic version
|
||||
CURRENT_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
|
||||
|
||||
# Check that the major part of pydantic version is as expected, if not
|
||||
# raise an error
|
||||
if [[ "$CURRENT_VERSION" != "$TEST_WITH_VERSION" ]]; then
|
||||
echo "Error: expected pydantic version ${CURRENT_VERSION} to have been installed, but found: ${TEST_WITH_VERSION}"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found pydantic version ${CURRENT_VERSION}, as expected"
|
||||
- name: Run pydantic compatibility tests
|
||||
shell: bash
|
||||
run: make test
|
||||
|
||||
- name: Ensure the tests did not create any additional files
|
||||
shell: bash
|
||||
run: |
|
||||
set -eu
|
||||
|
||||
STATUS="$(git status)"
|
||||
echo "$STATUS"
|
||||
|
||||
# grep will exit non-zero if the target message isn't found,
|
||||
# and `set -e` above will cause the step to fail.
|
||||
echo "$STATUS" | grep 'nothing to commit, working tree clean'
|
||||
@@ -1,63 +0,0 @@
|
||||
name: release
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
working-directory:
|
||||
required: true
|
||||
type: string
|
||||
description: "From which folder this pipeline executes"
|
||||
|
||||
env:
|
||||
POETRY_VERSION: "1.6.1"
|
||||
|
||||
jobs:
|
||||
if_release:
|
||||
# Disallow publishing from branches that aren't `main`.
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
# This permission is used for trusted publishing:
|
||||
# https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
|
||||
#
|
||||
# Trusted publishing has to also be configured on PyPI for each package:
|
||||
# https://docs.pypi.org/trusted-publishers/adding-a-publisher/
|
||||
id-token: write
|
||||
|
||||
# This permission is needed by `ncipollo/release-action` to create the GitHub release.
|
||||
contents: write
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: "3.10"
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
cache-key: release
|
||||
|
||||
- name: Build project for distribution
|
||||
run: poetry build
|
||||
- name: Check Version
|
||||
id: check-version
|
||||
run: |
|
||||
echo version=$(poetry version --short) >> $GITHUB_OUTPUT
|
||||
- name: Create Release
|
||||
uses: ncipollo/release-action@v1
|
||||
with:
|
||||
artifacts: "dist/*"
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
draft: false
|
||||
generateReleaseNotes: true
|
||||
tag: v${{ steps.check-version.outputs.version }}
|
||||
commit: main
|
||||
- name: Publish package distributions to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: ${{ inputs.working-directory }}/dist/
|
||||
verbose: true
|
||||
print-hash: true
|
||||
@@ -1,57 +0,0 @@
|
||||
name: test
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
working-directory:
|
||||
required: true
|
||||
type: string
|
||||
description: "From which folder this pipeline executes"
|
||||
|
||||
env:
|
||||
POETRY_VERSION: "1.6.1"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.8"
|
||||
- "3.9"
|
||||
- "3.10"
|
||||
- "3.11"
|
||||
name: Python ${{ matrix.python-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
cache-key: core
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: poetry install
|
||||
|
||||
- name: Run core tests
|
||||
shell: bash
|
||||
run: make test
|
||||
|
||||
- name: Ensure the tests did not create any additional files
|
||||
shell: bash
|
||||
run: |
|
||||
set -eu
|
||||
|
||||
STATUS="$(git status)"
|
||||
echo "$STATUS"
|
||||
|
||||
# grep will exit non-zero if the target message isn't found,
|
||||
# and `set -e` above will cause the step to fail.
|
||||
echo "$STATUS" | grep 'nothing to commit, working tree clean'
|
||||
@@ -1,150 +0,0 @@
|
||||
---
|
||||
name: Run CI Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- 'README.md'
|
||||
workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI
|
||||
|
||||
# If another push to the same PR or branch happens while this workflow is still running,
|
||||
# cancel the earlier run in favor of the next run.
|
||||
#
|
||||
# There's no point in testing an outdated version of the code. GitHub only allows
|
||||
# a limited number of job runners to be active at the same time, so it's better to cancel
|
||||
# pointless jobs early so that more useful jobs can run sooner.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
POETRY_VERSION: "1.5.1"
|
||||
WORKDIR: "."
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
uses:
|
||||
./.github/workflows/_lint.yml
|
||||
with:
|
||||
working-directory: .
|
||||
secrets: inherit
|
||||
|
||||
pydantic-compatibility:
|
||||
uses:
|
||||
./.github/workflows/_pydantic_compatibility.yml
|
||||
with:
|
||||
working-directory: .
|
||||
secrets: inherit
|
||||
test:
|
||||
timeout-minutes: 5
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ${{ env.WORKDIR }}
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.8"
|
||||
- "3.9"
|
||||
- "3.10"
|
||||
- "3.11"
|
||||
name: Python ${{ matrix.python-version }} tests
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: .
|
||||
cache-key: benchmarks-all
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Running tests, installing dependencies with poetry..."
|
||||
poetry install --with test,lint,typing,docs
|
||||
|
||||
- name: Run tests
|
||||
run: make test
|
||||
|
||||
- name: Ensure the tests did not create any additional files
|
||||
shell: bash
|
||||
run: |
|
||||
set -eu
|
||||
|
||||
STATUS="$(git status)"
|
||||
echo "$STATUS"
|
||||
|
||||
# grep will exit non-zero if the target message isn't found,
|
||||
# and `set -e` above will cause the step to fail.
|
||||
echo "$STATUS" | grep 'nothing to commit, working tree clean'
|
||||
test_docs:
|
||||
timeout-minutes: 5
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ${{ env.WORKDIR }}
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.11"
|
||||
name: Documentation Build for Python ${{ matrix.python-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: .
|
||||
cache-key: benchmarks-all
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Running tests, installing dependencies with poetry..."
|
||||
poetry install --with test,lint,typing,docs
|
||||
|
||||
- name: Test Sphinx Docs
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Attempting to build docs..."
|
||||
make docs_build
|
||||
test_datasets:
|
||||
timeout-minutes: 5
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ${{ env.WORKDIR }}
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.11"
|
||||
name: Validate Public Datasets
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: .
|
||||
cache-key: benchmarks-all
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Running tests, installing dependencies with poetry..."
|
||||
poetry install --with test,lint,typing,docs
|
||||
|
||||
- name: Request datasets
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Attempting to build docs..."
|
||||
poetry run python -m scripts.check_datasets
|
||||
@@ -1,44 +0,0 @@
|
||||
name: Publish Docs
|
||||
on: [workflow_dispatch]
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
env:
|
||||
POETRY_VERSION: "1.6.1"
|
||||
|
||||
jobs:
|
||||
docs:
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.11"
|
||||
runs-on: ubuntu-latest
|
||||
name: Documentation Publish
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: .
|
||||
cache-key: benchmarks-all
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Running tests, installing dependencies with poetry..."
|
||||
poetry install --with test,lint,typing,docs
|
||||
|
||||
- name: Sphinx build
|
||||
shell: bash
|
||||
run: |
|
||||
make docs_build
|
||||
- name: Publish Docs
|
||||
uses: peaceiris/actions-gh-pages@v3
|
||||
with:
|
||||
publish_branch: gh-pages
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
publish_dir: ./docs/build
|
||||
force_orphan: true
|
||||
@@ -1,14 +0,0 @@
|
||||
---
|
||||
name: Publish Package to PyPi
|
||||
|
||||
on:
|
||||
workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI
|
||||
|
||||
jobs:
|
||||
release:
|
||||
uses:
|
||||
./.github/workflows/_release.yml
|
||||
permissions: write-all
|
||||
with:
|
||||
working-directory: .
|
||||
secrets: inherit
|
||||
@@ -1,33 +0,0 @@
|
||||
name: Weekly Tool Benchmarks
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 0 * * 0' # Runs at midnight (00:00) every Sunday (UTC time)
|
||||
|
||||
jobs:
|
||||
run_tool_benchmarks:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.12 + Poetry ${{ env.POETRY_VERSION }}
|
||||
uses: "./.github/actions/poetry_setup"
|
||||
with:
|
||||
python-version: '3.12'
|
||||
poetry-version: ${{ env.POETRY_VERSION }}
|
||||
working-directory: .
|
||||
cache-key: benchmarks-all
|
||||
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Running tests, installing dependencies with poetry..."
|
||||
poetry install --with test,lint,typing,docs
|
||||
|
||||
- name: Multiverse math benchmark
|
||||
run: python scripts/multiverse_math_benchmark.py
|
||||
|
||||
- name: Query analysis benchmark
|
||||
run: python scripts/query_analysis_benchmark.py
|
||||
+2
-2
@@ -158,5 +158,5 @@ cython_debug/
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
.idea/
|
||||
.DS_Store
|
||||
#.idea/
|
||||
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Langchain AI
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,67 +0,0 @@
|
||||
.PHONY: all lint format test help
|
||||
|
||||
# Default target executed when no arguments are given to make.
|
||||
all: help
|
||||
|
||||
# LINTING AND FORMATTING:
|
||||
|
||||
# Define a variable for Python and notebook files.
|
||||
lint format: PYTHON_FILES=.
|
||||
lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=. --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
|
||||
|
||||
lint lint_diff:
|
||||
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES) --diff
|
||||
# [ "$(PYTHON_FILES)" = "" ] || poetry run mypy $(PYTHON_FILES)
|
||||
|
||||
format format_diff:
|
||||
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES)
|
||||
[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I --fix $(PYTHON_FILES)
|
||||
|
||||
spell_check:
|
||||
poetry run codespell --toml pyproject.toml
|
||||
|
||||
spell_fix:
|
||||
poetry run codespell --toml pyproject.toml -w
|
||||
|
||||
|
||||
# TESTING AND COVERAGE:
|
||||
|
||||
# Define a variable for the test file path.
|
||||
TEST_FILE ?= tests/unit_tests/
|
||||
|
||||
test:
|
||||
poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
|
||||
|
||||
test_watch:
|
||||
poetry run ptw . -- $(TEST_FILE)
|
||||
|
||||
|
||||
# DOCUMENTATION:
|
||||
|
||||
docs_clean:
|
||||
rm -rf ./docs/build
|
||||
|
||||
docs_build:
|
||||
# Copy README.md to docs/index.md
|
||||
cp README.md ./docs/source/index.md
|
||||
# Append to the table of contents the contents of the file
|
||||
cat ./docs/source/toc.segment >> ./docs/source/index.md
|
||||
poetry run sphinx-build "./docs/source" "./docs/build"
|
||||
|
||||
|
||||
# HELP:
|
||||
help:
|
||||
@echo ''
|
||||
@echo 'LINTING:'
|
||||
@echo ' format - run code formatters'
|
||||
@echo ' lint - run linters'
|
||||
@echo ' spell_check - run codespell'
|
||||
@echo ' spell_fix - run codespell and fix the errors'
|
||||
@echo 'TESTS:'
|
||||
@echo ' test - run unit tests'
|
||||
@echo ' test TEST_FILE=<test_file> - run tests in <test_file>'
|
||||
@echo ' coverage - run unit tests and generate coverage report'
|
||||
@echo 'DOCUMENTATION:'
|
||||
@echo ' docs_clean - delete the docs/build directory'
|
||||
@echo ' docs_build - build the documentation'
|
||||
@echo ''
|
||||
@@ -1,19 +1,8 @@
|
||||
# 🦜💯 LangChain Benchmarks
|
||||
# LangChain Benchmarks
|
||||
|
||||
[](https://github.com/langchain-ai/langchain-benchmarks/releases)
|
||||
[](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://twitter.com/langchainai)
|
||||
[](https://discord.gg/6adMQxSpJS)
|
||||
[](https://github.com/langchain-ai/langchain-benchmarks/issues)
|
||||
|
||||
|
||||
[📖 Documentation](https://langchain-ai.github.io/langchain-benchmarks/index.html)
|
||||
|
||||
A package to help benchmark various LLM related tasks.
|
||||
|
||||
The benchmarks are organized by end-to-end use cases, and
|
||||
utilize [LangSmith](https://smith.langchain.com/) heavily.
|
||||
This repository shows how we benchmark some of our more popular chains and agents.
|
||||
The benchmarks are organized by end-to-end use cases.
|
||||
They utilize [LangSmith](https://smith.langchain.com/) heavily.
|
||||
|
||||
We have several goals in open sourcing this:
|
||||
|
||||
@@ -22,62 +11,8 @@ We have several goals in open sourcing this:
|
||||
- Showing how we evaluate each task
|
||||
- Encouraging others to benchmark their solutions on these tasks (we are always looking for better ways of doing things!)
|
||||
|
||||
## Benchmarking Results
|
||||
|
||||
Read some of the articles about benchmarking results on our blog.
|
||||
|
||||
* [Agent Tool Use](https://blog.langchain.dev/benchmarking-agent-tool-use/)
|
||||
* [Query Analysis in High Cardinality Situations](https://blog.langchain.dev/high-cardinality/)
|
||||
* [RAG on Tables](https://blog.langchain.dev/benchmarking-rag-on-tables/)
|
||||
* [Q&A over CSV data](https://blog.langchain.dev/benchmarking-question-answering-over-csv-data/)
|
||||
|
||||
|
||||
### Tool Usage (2024-04-18)
|
||||
|
||||
See [tool usage docs](https://langchain-ai.github.io/langchain-benchmarks/notebooks/tool_usage/benchmark_all_tasks.html) to recreate!
|
||||
|
||||
|
||||
|
||||
Explore Agent Traces on LangSmith:
|
||||
|
||||
* [Relational Data](https://smith.langchain.com/public/22721064-dcf6-4e42-be65-e7c46e6835e7/d)
|
||||
* [Tool Usage (1-tool)](https://smith.langchain.com/public/ac23cb40-e392-471f-b129-a893a77b6f62/d)
|
||||
* [Tool Usage (26-tools)](https://smith.langchain.com/public/366bddca-62b3-4b6e-849b-a478abab73db/d)
|
||||
* [Mutiverse Math](https://smith.langchain.com/public/983faff2-54b9-4875-9bf2-c16913e7d489/d)
|
||||
|
||||
## Installation
|
||||
|
||||
To install the packages, run the following command:
|
||||
|
||||
```bash
|
||||
pip install -U langchain-benchmarks
|
||||
```
|
||||
|
||||
All the benchmarks come with an associated benchmark dataset stored in [LangSmith](https://smith.langchain.com). To take advantage of the eval and debugging experience, [sign up](https://smith.langchain.com), and set your API key in your environment:
|
||||
|
||||
```bash
|
||||
export LANGCHAIN_API_KEY=ls-...
|
||||
```
|
||||
|
||||
## Repo Structure
|
||||
|
||||
The package is located within [langchain_benchmarks](./langchain_benchmarks/). Check out the [docs](https://langchain-ai.github.io/langchain-benchmarks/index.html) for information on how to get starte.
|
||||
|
||||
The other directories are legacy and may be moved in the future.
|
||||
|
||||
|
||||
## Archived
|
||||
|
||||
Below are archived benchmarks that require cloning this repo to run.
|
||||
|
||||
- [CSV Question Answering](https://github.com/langchain-ai/langchain-benchmarks/tree/main/archived/csv-qa)
|
||||
- [Extraction](https://github.com/langchain-ai/langchain-benchmarks/tree/main/archived/extraction)
|
||||
- [Q&A over the LangChain docs](https://github.com/langchain-ai/langchain-benchmarks/tree/main/archived/langchain-docs-benchmarking)
|
||||
- [Meta-evaluation of 'correctness' evaluators](https://github.com/langchain-ai/langchain-benchmarks/tree/main/archived/meta-evals)
|
||||
|
||||
|
||||
## Related
|
||||
|
||||
- For cookbooks on other ways to test, debug, monitor, and improve your LLM applications, check out the [LangSmith docs](https://docs.smith.langchain.com/)
|
||||
- For information on building with LangChain, check out the [python documentation](https://python.langchain.com/docs/get_started/introduction) or [JS documentation](https://js.langchain.com/docs/get_started/introduction)
|
||||
|
||||
We currently include the following tasks:
|
||||
- [CSV Question Answering](csv-qa)
|
||||
- [Extraction](extraction)
|
||||
- [Q&A over the LangChain docs](langchain-docs-benchmarking)
|
||||
- [Meta-evaluation of 'correctness' evaluators](meta-evals)
|
||||
@@ -0,0 +1,107 @@
|
||||
# Testing Agents
|
||||
|
||||
This directory contains environments that can be used to test agent's ability
|
||||
to use tools and make decisions.
|
||||
|
||||
## Environments
|
||||
|
||||
Environments are named in the style of e[env_number]_[name].py.
|
||||
|
||||
### e01_alpha
|
||||
|
||||
* Consists of 3 relational tables of users, locations and foods.
|
||||
* Defines a set of tools that can be used these tables.
|
||||
* Agent should use the given tools to answer questions.
|
||||
|
||||
## Running Evaluation
|
||||
|
||||
### Install dependencies
|
||||
|
||||
```bash
|
||||
poetry install
|
||||
```
|
||||
|
||||
### Run evaluation
|
||||
|
||||
We'll make this more convenient in the future, but for now, you can run
|
||||
|
||||
|
||||
```python
|
||||
from langchain.agents.format_scratchpad import format_to_openai_functions
|
||||
from langchain.agents import AgentExecutor
|
||||
from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
|
||||
from langchain.tools.render import format_tool_to_openai_function
|
||||
|
||||
from environments.e01_alpha import get_tools
|
||||
|
||||
|
||||
def agent_factory() -> AgentExecutor:
|
||||
"""This creates an OpenAI agent."""
|
||||
llm = ChatOpenAI(
|
||||
model="gpt-3.5-turbo-16k",
|
||||
temperature=0,
|
||||
)
|
||||
tools = get_tools()
|
||||
llm_with_tools = llm.bind(
|
||||
functions=[format_tool_to_openai_function(t) for t in tools]
|
||||
)
|
||||
prompt = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
("system", "You are a helpful assistant. Use the given tools to answer the question. Keep in mind that an ID is distinct from a name for every entity."),
|
||||
MessagesPlaceholder(variable_name="agent_scratchpad"),
|
||||
("user", "{input}"),
|
||||
]
|
||||
)
|
||||
|
||||
runnable_agent = (
|
||||
{
|
||||
"input": lambda x: x["question"],
|
||||
"agent_scratchpad": lambda x: format_to_openai_functions(
|
||||
x["intermediate_steps"]
|
||||
),
|
||||
}
|
||||
| prompt
|
||||
| llm_with_tools
|
||||
| OpenAIFunctionsAgentOutputParser()
|
||||
)
|
||||
|
||||
return AgentExecutor(
|
||||
agent=runnable_agent,
|
||||
tools=tools,
|
||||
handle_parsing_errors=True,
|
||||
return_intermediate_steps=True,
|
||||
)
|
||||
```
|
||||
|
||||
Confirm that the agent works and can use the environment tools:
|
||||
|
||||
```python
|
||||
agent_factory().invoke({'question': "who is bob?"})
|
||||
```
|
||||
|
||||
```python
|
||||
from langsmith import Client
|
||||
|
||||
from environments.e01_alpha import DATASET_ID
|
||||
from evaluators import STANDARD_AGENT_EVALUATOR
|
||||
|
||||
client = Client()
|
||||
|
||||
dataset_name = client.read_dataset(dataset_id=DATASET_ID).name
|
||||
|
||||
results = client.run_on_dataset(
|
||||
dataset_name=dataset_name,
|
||||
llm_or_chain_factory=agent_factory,
|
||||
evaluation=STANDARD_AGENT_EVALUATOR,
|
||||
verbose=True,
|
||||
tags=["openai-agent", "gpt-3.5-turbo-16k"],
|
||||
)
|
||||
```
|
||||
|
||||
|
||||
### Customize evaluation
|
||||
|
||||
Please refer to the following example to see how to set up and run evaluation
|
||||
for agents using [LangSmith](https://github.com/langchain-ai/langsmith-cookbook/blob/main/testing-examples/agent_steps/evaluating_agents.ipynb).
|
||||
@@ -0,0 +1 @@
|
||||
"""Package for helping to evaluate agent runs."""
|
||||
+21
-63
@@ -1,19 +1,20 @@
|
||||
"""Answer questions about relational data using the provided tools.
|
||||
"""A simple environment for evaluating an agent.
|
||||
|
||||
A simple environment to evaluate an agent's ability to use a set of given tools
|
||||
to reference questions.
|
||||
|
||||
The environment contains fake data about users and their locations and favorite foods.
|
||||
|
||||
The environment provides a set of tools that can be used to query the data.
|
||||
The environment defines a set of tools that the agent can use to access the data.
|
||||
|
||||
All questions can be answered by using the provided tools. The answers
|
||||
include the expected result as well as the most efficient way to answer the
|
||||
question using the tools.
|
||||
Agent performance should be evaluated solely based on the agent's ability to use
|
||||
the tools to reference questions.
|
||||
"""
|
||||
from typing import Callable, List, TypedDict
|
||||
from typing import List, Callable
|
||||
from typing import TypedDict
|
||||
|
||||
from langchain.tools import StructuredTool
|
||||
from langchain_core.tools import ToolException
|
||||
|
||||
from langchain_benchmarks.schema import ToolUsageEnvironment, ToolUsageTask
|
||||
from langchain.tools import BaseTool
|
||||
from langchain.tools import tool
|
||||
|
||||
USER_DATA = [
|
||||
# IDs are not consecutive to prevent agents from guessing the ID
|
||||
@@ -147,10 +148,11 @@ FOOD_DATA = [
|
||||
]
|
||||
|
||||
|
||||
class SearchHit(TypedDict, total=False):
|
||||
class SearchHit(TypedDict):
|
||||
"""A search hit."""
|
||||
|
||||
id: str
|
||||
value: str
|
||||
|
||||
|
||||
def _similarity_search(data: List[dict], query: str, key: str) -> List[SearchHit]:
|
||||
@@ -167,12 +169,8 @@ def _similarity_search(data: List[dict], query: str, key: str) -> List[SearchHit
|
||||
Returns:
|
||||
The list of matching data.
|
||||
"""
|
||||
|
||||
def _score_function(x: str) -> float:
|
||||
"""Calculate the similarity score between the query and the given string."""
|
||||
return len(set(x) & set(query)) / len(set(x) | set(query))
|
||||
|
||||
re_ranked_data = sorted(data, key=lambda x: _score_function(x[key]), reverse=True)
|
||||
score_function = lambda x: len(set(x) & set(query)) / len(set(x) | set(query))
|
||||
re_ranked_data = sorted(data, key=lambda x: score_function(x[key]), reverse=True)
|
||||
return [{"id": d["id"], key: d[key]} for d in re_ranked_data]
|
||||
|
||||
|
||||
@@ -188,7 +186,7 @@ def _get_user(id: int) -> dict:
|
||||
for user in USER_DATA:
|
||||
if user["id"] == id:
|
||||
return user
|
||||
raise ToolException(f"User ID {id} cannot be resolved")
|
||||
raise ValueError(f"User ID {id} cannot be resolved")
|
||||
|
||||
|
||||
def _get_location(id: int) -> dict:
|
||||
@@ -203,7 +201,7 @@ def _get_location(id: int) -> dict:
|
||||
for location in LOCATION_DATA:
|
||||
if location["id"] == id:
|
||||
return location
|
||||
raise ToolException(f"Location ID {id} cannot be resolved")
|
||||
raise ValueError(f"Location ID {id} cannot be resolved")
|
||||
|
||||
|
||||
def _get_food(food_id: int) -> dict:
|
||||
@@ -218,7 +216,7 @@ def _get_food(food_id: int) -> dict:
|
||||
for food in FOOD_DATA:
|
||||
if food["id"] == food_id:
|
||||
return food
|
||||
raise ToolException(f"Food ID {food_id} cannot be resolved")
|
||||
raise ValueError(f"Food ID {food_id} cannot be resolved")
|
||||
|
||||
|
||||
def get_available_functions() -> List[Callable]:
|
||||
@@ -392,51 +390,11 @@ def get_available_functions() -> List[Callable]:
|
||||
return functions
|
||||
|
||||
|
||||
def get_tools() -> List[StructuredTool]:
|
||||
def get_tools() -> List[BaseTool]:
|
||||
"""Get all the available tools."""
|
||||
functions = get_available_functions()
|
||||
return [StructuredTool.from_function(f, handle_tool_error=True) for f in functions]
|
||||
|
||||
|
||||
def get_environment() -> ToolUsageEnvironment:
|
||||
"""Create an environment."""
|
||||
return ToolUsageEnvironment(
|
||||
tools=get_tools(),
|
||||
read_state=None,
|
||||
)
|
||||
return [tool(f) for f in functions]
|
||||
|
||||
|
||||
# ID of a dataset that contains the questions and references
|
||||
DATASET_ID = "https://smith.langchain.com/public/1d89f4b3-5f73-48cf-a127-2fdeb22f6d84/d"
|
||||
|
||||
RELATIONAL_DATA_TASK = ToolUsageTask(
|
||||
name="Tool Usage - Relational Data",
|
||||
dataset_id=DATASET_ID,
|
||||
create_environment=get_environment,
|
||||
instructions=(
|
||||
"""\
|
||||
Please answer the user's question by using the tools provided. Do not guess the \
|
||||
answer. Keep in mind that entities like users,foods and locations have both a \
|
||||
name and an ID, which are not the same."""
|
||||
),
|
||||
description=(
|
||||
"""\
|
||||
Environment with fake data about users and their locations and favorite foods.
|
||||
|
||||
The environment provides a set of tools that can be used to query the data.
|
||||
|
||||
The objective of this task is to evaluate the ability to use the provided tools \
|
||||
to answer questions about relational data.
|
||||
|
||||
The dataset contains 21 examples of varying difficulty. The difficulty is measured \
|
||||
by the number of tools that need to be used to answer the question.
|
||||
|
||||
Each example is composed of a question, a reference answer, and \
|
||||
information about the sequence in which tools should be used to answer \
|
||||
the question.
|
||||
|
||||
Success is measured by the ability to answer the question correctly, and efficiently.
|
||||
"""
|
||||
),
|
||||
eval_params={}, # No special evaluation parameters
|
||||
)
|
||||
DATASET_ID = "9f73165c-d333-4d14-8f59-bd7eede5db08" # ID of Agent Gym: E01 Alpha
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Module contains standard evaluators for agents.
|
||||
|
||||
Requirements:
|
||||
|
||||
* Agents must output "intermediate_steps" in their run outputs.
|
||||
* The dataset must have "expected_steps" in its outputs.
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
from langchain.evaluation import EvaluatorType
|
||||
from langchain.smith import RunEvalConfig
|
||||
from langsmith.evaluation.evaluator import (
|
||||
EvaluationResults,
|
||||
RunEvaluator,
|
||||
EvaluationResult,
|
||||
)
|
||||
from langsmith.schemas import Example, Run
|
||||
|
||||
|
||||
class AgentTrajectoryEvaluator(RunEvaluator):
|
||||
"""An evaluator that can be used in conjunction with a standard agent interface."""
|
||||
|
||||
def evaluate_run(
|
||||
self, run: Run, example: Optional[Example] = None
|
||||
) -> EvaluationResults:
|
||||
if run.outputs is None:
|
||||
raise ValueError("Run outputs cannot be None")
|
||||
# This is the output of each run
|
||||
intermediate_steps = run.outputs["intermediate_steps"]
|
||||
# Since we are comparing to the tool names, we now need to get that
|
||||
# Intermediate steps is a Tuple[AgentAction, Any]
|
||||
# The first element is the action taken
|
||||
# The second element is the observation from taking that action
|
||||
trajectory = [action.tool for action, _ in intermediate_steps]
|
||||
# This is what we uploaded to the dataset
|
||||
expected_trajectory = example.outputs["expected_steps"]
|
||||
|
||||
# Just score it based on whether it is correct or not
|
||||
score = int(trajectory == expected_trajectory)
|
||||
step_fraction = len(trajectory) / len(expected_trajectory)
|
||||
|
||||
return {
|
||||
"results": [
|
||||
EvaluationResult(
|
||||
key="Intermediate steps correctness",
|
||||
score=score,
|
||||
),
|
||||
EvaluationResult(
|
||||
key="# steps / # expected steps",
|
||||
score=step_fraction,
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
STANDARD_AGENT_EVALUATOR = RunEvalConfig(
|
||||
# Evaluators can either be an evaluator type
|
||||
# (e.g., "qa", "criteria", "embedding_distance", etc.) or a
|
||||
# configuration for that evaluator
|
||||
evaluators=[
|
||||
# Measures whether a QA response is "Correct", based on a reference answer
|
||||
# You can also select via the raw string "qa"
|
||||
EvaluatorType.QA
|
||||
],
|
||||
# You can add custom StringEvaluator or RunEvaluator objects
|
||||
# here as well, which will automatically be
|
||||
# applied to each prediction. Check out the docs for examples.
|
||||
custom_evaluators=[AgentTrajectoryEvaluator()],
|
||||
# We now need to specify this because we have multiple outputs in our dataset
|
||||
reference_key="reference",
|
||||
)
|
||||
Generated
+1177
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,16 @@
|
||||
[tool.poetry]
|
||||
name = "agents"
|
||||
version = "0.1.0"
|
||||
description = "Agent Benchmarking Environments"
|
||||
authors = ["Eugene Yurtsev <eyurtsev@gmail.com>"]
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.8.1"
|
||||
langchain = "^0.0.335"
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
@@ -1,45 +0,0 @@
|
||||
import pandas as pd
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts import ChatPromptTemplate
|
||||
from langchain.schema.output_parser import StrOutputParser
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
from langsmith import Client
|
||||
from pandasai import PandasAI
|
||||
|
||||
if __name__ == "__main__":
|
||||
df = pd.read_csv("titanic.csv")
|
||||
|
||||
pandas_ai = PandasAI(ChatOpenAI(temperature=0, model="gpt-4"), enable_cache=False)
|
||||
prompt = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
(
|
||||
"system",
|
||||
"Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer",
|
||||
),
|
||||
("human", "Question: {input}"),
|
||||
("human", "Data Scientist Result: {result}"),
|
||||
]
|
||||
)
|
||||
|
||||
def get_chain():
|
||||
chain = (
|
||||
{
|
||||
"input": lambda x: x["input_question"],
|
||||
"result": lambda x: pandas_ai(df, prompt=x["input_question"]),
|
||||
}
|
||||
| prompt
|
||||
| ChatOpenAI(temperature=0, model="gpt-4")
|
||||
| StrOutputParser()
|
||||
)
|
||||
return chain
|
||||
|
||||
client = Client()
|
||||
eval_config = RunEvalConfig(
|
||||
evaluators=["qa"],
|
||||
)
|
||||
chain_results = run_on_dataset(
|
||||
client,
|
||||
dataset_name="Titanic CSV Data",
|
||||
llm_or_chain_factory=get_chain,
|
||||
evaluation=eval_config,
|
||||
)
|
||||
@@ -1,47 +0,0 @@
|
||||
import pandas as pd
|
||||
import streamlit as st
|
||||
from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
|
||||
from langchain.agents.agent_types import AgentType
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
|
||||
df = pd.read_csv("titanic.csv")
|
||||
|
||||
|
||||
llm = ChatOpenAI(temperature=0)
|
||||
agent = create_pandas_dataframe_agent(llm, df, agent_type=AgentType.OPENAI_FUNCTIONS)
|
||||
|
||||
|
||||
from langsmith import Client
|
||||
|
||||
client = Client()
|
||||
|
||||
|
||||
def send_feedback(run_id, score):
|
||||
client.create_feedback(run_id, "user_score", score=score)
|
||||
|
||||
|
||||
st.set_page_config(page_title="🦜🔗 Ask the CSV App")
|
||||
st.title("🦜🔗 Ask the CSV App")
|
||||
st.info(
|
||||
"Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!"
|
||||
)
|
||||
|
||||
query_text = st.text_input("Enter your question:", placeholder="Who was in cabin C128?")
|
||||
# Form input and query
|
||||
result = None
|
||||
with st.form("myform", clear_on_submit=True):
|
||||
submitted = st.form_submit_button("Submit")
|
||||
if submitted:
|
||||
with st.spinner("Calculating..."):
|
||||
response = agent({"input": query_text}, include_run_info=True)
|
||||
result = response["output"]
|
||||
run_id = response["__run"].run_id
|
||||
if result is not None:
|
||||
st.info(result)
|
||||
col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
|
||||
with col_text:
|
||||
st.text("Feedback:")
|
||||
with col1:
|
||||
st.button("👍", on_click=send_feedback, args=(run_id, 1))
|
||||
with col2:
|
||||
st.button("👎", on_click=send_feedback, args=(run_id, 0))
|
||||
@@ -1,79 +0,0 @@
|
||||
import streamlit as st
|
||||
from langchain.chains import create_extraction_chain
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langsmith import Client
|
||||
|
||||
st.set_page_config(page_title="🦜🔗 Text-to-graph extraction")
|
||||
client = Client()
|
||||
|
||||
|
||||
def send_feedback(run_id, score):
|
||||
client.create_feedback(run_id, "user_score", score=score)
|
||||
|
||||
|
||||
st.title("🦜🔗 Text-to-graph playground")
|
||||
st.info(
|
||||
"This playground explores the use of [OpenAI functions](https://openai.com/blog/function-calling-and-other-api-updates) and [LangChain](https://github.com/langchain-ai/langchain) to build knowledge graphs from user-input text. It breaks down the user input text into knowledge graph triples of subject (primary entities or concepts in a sentence), predicate (actions or relationships that connect subjects to objects), and object (entities or concepts that interact with or are acted upon by the subjects)."
|
||||
)
|
||||
|
||||
# Input text (optional default)
|
||||
oppenheimer_text = """'Julius Robert Oppenheimer, often known as Robert or "Oppie", is heralded as the father of the atomic bomb. Emerging from a non-practicing Jewish family in New York, he made several breakthroughs, such as the early black hole theory, before the monumental Manhattan Project. His wife, Katherine “Kitty” Oppenheimer, was a German-born woman with a complex past, including connections to the Communist Party. Oppenheimer\'s journey was beset by political adversaries, notably Lewis Strauss, chairman of the U.S. Atomic Energy Commission, and William Borden, an executive director with hawkish nuclear ambitions. These tensions culminated in the famous 1954 security hearing. Influential figures like lieutenant general Leslie Groves, who had also overseen the Pentagon\'s creation, stood by Oppenheimer\'s side, having earlier chosen him for the Manhattan Project and the Los Alamos location. Intimate relationships, like that with Jean Tatlock, a Communist and the possible muse behind the Trinity test\'s name, and colleagues like Frank, Oppenheimer\'s physicist brother, intertwined with his professional life. Scientists such as Ernest Lawrence, Edward Teller, David Hill, Richard Feynman, and Hans Bethe were some of Oppenheimer\'s contemporaries, each contributing to and contesting the atomic age\'s directions. Boris Pash\'s investigations, and the perspectives of figures like Leo Szilard, Niels Bohr, Harry Truman, and others, framed the broader sociopolitical context. Meanwhile, individuals like Robert Serber, Enrico Fermi, Albert Einstein, and Isidor Isaac Rabi, among many others, each played their parts in this narrative, from naming the atomic bombs to pivotal scientific contributions and advisory roles. All these figures, together with the backdrop of World War II, McCarthyism, and the dawn of the nuclear age, presented a complex mosaic of ambitions, loyalties, betrayals, and ideologies.oppenheimer_short.txt"""
|
||||
|
||||
# Knowledge triplet schema
|
||||
default_schema = {
|
||||
"properties": {
|
||||
"subject": {"type": "string"},
|
||||
"predicate": {"type": "string"},
|
||||
"object": {"type": "string"},
|
||||
},
|
||||
"required": ["subject", "predicate", "object"],
|
||||
}
|
||||
|
||||
# Create a text_area, set the default value to oppenheimer_text
|
||||
MAX_CHARS = 2000 # Maximum number of characters
|
||||
user_input_text = st.text_area("Enter your text (<2000 characters):", height=200)
|
||||
if len(user_input_text) > MAX_CHARS:
|
||||
st.warning(f"Text is too long. Processing only the first {MAX_CHARS} characters")
|
||||
user_input_text = user_input_text[:MAX_CHARS]
|
||||
|
||||
|
||||
# Output formatting of triples
|
||||
def json_to_markdown_table(json_list):
|
||||
if not json_list:
|
||||
return "No data available."
|
||||
|
||||
# Extract headers
|
||||
headers = json_list[0].keys()
|
||||
markdown_table = " | ".join(headers) + "\n"
|
||||
markdown_table += " | ".join(["---"] * len(headers)) + "\n"
|
||||
|
||||
# Extract rows
|
||||
for item in json_list:
|
||||
row = " | ".join([str(item[header]) for header in headers])
|
||||
markdown_table += row + "\n"
|
||||
|
||||
return markdown_table
|
||||
|
||||
|
||||
# Form input and query
|
||||
markdown_output = None
|
||||
with st.form("myform", clear_on_submit=True):
|
||||
submitted = st.form_submit_button("Submit")
|
||||
if submitted:
|
||||
with st.spinner("Calculating..."):
|
||||
llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo")
|
||||
chain = create_extraction_chain(default_schema, llm)
|
||||
extraction_output = chain(user_input_text, include_run_info=True)
|
||||
markdown_output = json_to_markdown_table(extraction_output["text"])
|
||||
run_id = extraction_output["__run"].run_id
|
||||
|
||||
# Feeback
|
||||
if markdown_output is not None:
|
||||
st.markdown(markdown_output)
|
||||
col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
|
||||
with col_text:
|
||||
st.text("Feedback:")
|
||||
with col1:
|
||||
st.button("👍", on_click=send_feedback, args=(run_id, 1))
|
||||
with col2:
|
||||
st.button("👎", on_click=send_feedback, args=(run_id, 0))
|
||||
@@ -1,57 +0,0 @@
|
||||
"""Copy the public dataset to your own langsmith tenant."""
|
||||
from typing import Optional
|
||||
|
||||
from langsmith import Client
|
||||
|
||||
DATASET_NAME = "LangChain Docs Q&A"
|
||||
PUBLIC_DATASET_TOKEN = "452ccafc-18e1-4314-885b-edd735f17b9d"
|
||||
|
||||
|
||||
def create_langchain_docs_dataset(
|
||||
dataset_name: str = DATASET_NAME,
|
||||
public_dataset_token: str = PUBLIC_DATASET_TOKEN,
|
||||
client: Optional[Client] = None,
|
||||
):
|
||||
shared_client = Client(
|
||||
api_url="https://api.smith.langchain.com", api_key="placeholder"
|
||||
)
|
||||
examples = list(shared_client.list_shared_examples(public_dataset_token))
|
||||
client = client or Client()
|
||||
if client.has_dataset(dataset_name=dataset_name):
|
||||
loaded_examples = list(client.list_examples(dataset_name=dataset_name))
|
||||
if len(loaded_examples) == len(examples):
|
||||
return
|
||||
else:
|
||||
ds = client.read_dataset(dataset_name=dataset_name)
|
||||
else:
|
||||
ds = client.create_dataset(dataset_name=dataset_name)
|
||||
client.create_examples(
|
||||
inputs=[e.inputs for e in examples],
|
||||
outputs=[e.outputs for e in examples],
|
||||
dataset_id=ds.id,
|
||||
)
|
||||
print("Done creating dataset.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--target-api-key", type=str, required=False)
|
||||
parser.add_argument("--target-endpoint", type=str, required=False)
|
||||
parser.add_argument("--dataset-name", type=str, default=DATASET_NAME)
|
||||
parser.add_argument(
|
||||
"--public-dataset-token", type=str, default=PUBLIC_DATASET_TOKEN
|
||||
)
|
||||
args = parser.parse_args()
|
||||
client = None
|
||||
if args.target_api_key or args.target_endpoint:
|
||||
client = Client(
|
||||
api_key=args.target_api_key,
|
||||
api_url=args.target_endpoint,
|
||||
)
|
||||
create_langchain_docs_dataset(
|
||||
dataset_name=args.dataset_name,
|
||||
public_dataset_token=args.public_dataset_token,
|
||||
client=client,
|
||||
)
|
||||
@@ -1,25 +1,22 @@
|
||||
import pandas as pd
|
||||
from langchain.agents import AgentExecutor, OpenAIFunctionsAgent
|
||||
from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
|
||||
create_retriever_tool,
|
||||
)
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.embeddings import OpenAIEmbeddings
|
||||
from langchain.agents import OpenAIFunctionsAgent, AgentExecutor
|
||||
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
from langchain.tools import PythonAstREPLTool
|
||||
from langchain.vectorstores import FAISS
|
||||
import pandas as pd
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langsmith import Client
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
from pydantic import BaseModel, Field
|
||||
from langchain.embeddings import OpenAIEmbeddings
|
||||
from langchain.vectorstores import FAISS
|
||||
from langchain.agents.agent_toolkits.conversational_retrieval.tool import create_retriever_tool
|
||||
|
||||
pd.set_option("display.max_rows", 20)
|
||||
pd.set_option("display.max_columns", 20)
|
||||
|
||||
pd.set_option('display.max_rows', 20)
|
||||
pd.set_option('display.max_columns', 20)
|
||||
|
||||
embedding_model = OpenAIEmbeddings()
|
||||
vectorstore = FAISS.load_local("titanic_data", embedding_model)
|
||||
retriever_tool = create_retriever_tool(
|
||||
vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
|
||||
)
|
||||
retriever_tool = create_retriever_tool(vectorstore.as_retriever(), "person_name_search", "Search for a person by name")
|
||||
|
||||
|
||||
TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
|
||||
@@ -45,6 +42,7 @@ For example:
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class PythonInputs(BaseModel):
|
||||
query: str = Field(description="code snippet to run")
|
||||
|
||||
@@ -53,33 +51,27 @@ if __name__ == "__main__":
|
||||
df = pd.read_csv("titanic.csv")
|
||||
template = TEMPLATE.format(dhead=df.head().to_markdown())
|
||||
|
||||
prompt = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
("system", template),
|
||||
MessagesPlaceholder(variable_name="agent_scratchpad"),
|
||||
("human", "{input}"),
|
||||
]
|
||||
)
|
||||
prompt = ChatPromptTemplate.from_messages([
|
||||
("system", template),
|
||||
MessagesPlaceholder(variable_name="agent_scratchpad"),
|
||||
("human", "{input}")
|
||||
])
|
||||
|
||||
def get_chain():
|
||||
repl = PythonAstREPLTool(
|
||||
locals={"df": df},
|
||||
name="python_repl",
|
||||
description="Runs code and returns the output of the final line",
|
||||
args_schema=PythonInputs,
|
||||
)
|
||||
repl = PythonAstREPLTool(locals={"df": df}, name="python_repl",
|
||||
description="Runs code and returns the output of the final line",
|
||||
args_schema=PythonInputs)
|
||||
tools = [repl, retriever_tool]
|
||||
agent = OpenAIFunctionsAgent(
|
||||
llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools
|
||||
)
|
||||
agent_executor = AgentExecutor(
|
||||
agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
|
||||
)
|
||||
agent = OpenAIFunctionsAgent(llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools)
|
||||
agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate")
|
||||
return agent_executor
|
||||
|
||||
|
||||
client = Client()
|
||||
eval_config = RunEvalConfig(
|
||||
evaluators=["qa"],
|
||||
evaluators=[
|
||||
"qa"
|
||||
],
|
||||
)
|
||||
chain_results = run_on_dataset(
|
||||
client,
|
||||
@@ -1,9 +1,9 @@
|
||||
import pandas as pd
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
|
||||
from langchain.agents.agent_types import AgentType
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
from langsmith import Client
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
|
||||
if __name__ == "__main__":
|
||||
df = pd.read_csv("titanic.csv")
|
||||
@@ -18,17 +18,20 @@ if __name__ == "__main__":
|
||||
df,
|
||||
agent_type=AgentType.OPENAI_FUNCTIONS,
|
||||
agent_executor_kwargs=agent_executor_kwargs,
|
||||
max_iterations=5,
|
||||
max_iterations=5
|
||||
)
|
||||
return agent
|
||||
|
||||
|
||||
client = Client()
|
||||
eval_config = RunEvalConfig(
|
||||
evaluators=["qa"],
|
||||
evaluators=[
|
||||
"qa"
|
||||
],
|
||||
)
|
||||
chain_results = run_on_dataset(
|
||||
client,
|
||||
dataset_name="Titanic CSV Data",
|
||||
llm_or_chain_factory=get_chain,
|
||||
evaluation=eval_config,
|
||||
)
|
||||
)
|
||||
@@ -1,13 +1,14 @@
|
||||
import pandas as pd
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
|
||||
from langchain.agents.agent_types import AgentType
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
from langsmith import Client
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
|
||||
if __name__ == "__main__":
|
||||
df = pd.read_csv("titanic.csv")
|
||||
|
||||
|
||||
def get_chain():
|
||||
llm = ChatOpenAI(temperature=0, model="gpt-4")
|
||||
agent_executor_kwargs = {
|
||||
@@ -18,17 +19,20 @@ if __name__ == "__main__":
|
||||
df,
|
||||
agent_type=AgentType.OPENAI_FUNCTIONS,
|
||||
agent_executor_kwargs=agent_executor_kwargs,
|
||||
max_iterations=5,
|
||||
max_iterations=5
|
||||
)
|
||||
return agent
|
||||
|
||||
|
||||
client = Client()
|
||||
eval_config = RunEvalConfig(
|
||||
evaluators=["qa"],
|
||||
evaluators=[
|
||||
"qa"
|
||||
],
|
||||
)
|
||||
chain_results = run_on_dataset(
|
||||
client,
|
||||
dataset_name="Titanic CSV Data",
|
||||
llm_or_chain_factory=get_chain,
|
||||
evaluation=eval_config,
|
||||
)
|
||||
)
|
||||
@@ -1,24 +1,22 @@
|
||||
import pandas as pd
|
||||
from langchain.agents import AgentExecutor, ZeroShotAgent
|
||||
from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
|
||||
create_retriever_tool,
|
||||
)
|
||||
from langchain.embeddings import OpenAIEmbeddings
|
||||
from langchain.llms import OpenAI
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
from langchain.agents import ZeroShotAgent, AgentExecutor
|
||||
from langchain.prompts import PromptTemplate
|
||||
from langchain.tools import PythonAstREPLTool
|
||||
from langchain.vectorstores import FAISS
|
||||
import pandas as pd
|
||||
from langchain.llms import OpenAI
|
||||
from langsmith import Client
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
from pydantic import BaseModel, Field
|
||||
from langchain.embeddings import OpenAIEmbeddings
|
||||
from langchain.vectorstores import FAISS
|
||||
from langchain.agents.agent_toolkits.conversational_retrieval.tool import create_retriever_tool
|
||||
|
||||
pd.set_option("display.max_rows", 20)
|
||||
pd.set_option("display.max_columns", 20)
|
||||
|
||||
pd.set_option('display.max_rows', 20)
|
||||
pd.set_option('display.max_columns', 20)
|
||||
|
||||
embedding_model = OpenAIEmbeddings()
|
||||
vectorstore = FAISS.load_local("titanic_data", embedding_model)
|
||||
retriever_tool = create_retriever_tool(
|
||||
vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
|
||||
)
|
||||
retriever_tool = create_retriever_tool(vectorstore.as_retriever(), "person_name_search", "Search for a person by name")
|
||||
|
||||
|
||||
TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
|
||||
@@ -43,6 +41,7 @@ For example:
|
||||
<logic>Use `python_repl` since even though the question is about a person, you don't know their name so you can't include it.</logic>"""
|
||||
|
||||
|
||||
|
||||
class PythonInputs(BaseModel):
|
||||
query: str = Field(description="code snippet to run")
|
||||
|
||||
@@ -51,27 +50,22 @@ if __name__ == "__main__":
|
||||
df = pd.read_csv("titanic.csv")
|
||||
template = TEMPLATE.format(dhead=df.head().to_markdown())
|
||||
|
||||
|
||||
def get_chain():
|
||||
repl = PythonAstREPLTool(
|
||||
locals={"df": df},
|
||||
name="python_repl",
|
||||
description="Runs code and returns the output of the final line",
|
||||
args_schema=PythonInputs,
|
||||
)
|
||||
repl = PythonAstREPLTool(locals={"df": df}, name="python_repl",
|
||||
description="Runs code and returns the output of the final line",
|
||||
args_schema=PythonInputs)
|
||||
tools = [repl, retriever_tool]
|
||||
agent = ZeroShotAgent.from_llm_and_tools(
|
||||
llm=OpenAI(temperature=0, model="gpt-3.5-turbo-instruct"),
|
||||
tools=tools,
|
||||
prefix=template,
|
||||
)
|
||||
agent_executor = AgentExecutor(
|
||||
agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
|
||||
)
|
||||
agent = ZeroShotAgent.from_llm_and_tools(llm=OpenAI(temperature=0, model="gpt-3.5-turbo-instruct"), tools=tools, prefix=template)
|
||||
agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate")
|
||||
return agent_executor
|
||||
|
||||
|
||||
client = Client()
|
||||
eval_config = RunEvalConfig(
|
||||
evaluators=["qa"],
|
||||
evaluators=[
|
||||
"qa"
|
||||
],
|
||||
)
|
||||
chain_results = run_on_dataset(
|
||||
client,
|
||||
@@ -0,0 +1,44 @@
|
||||
import pandas as pd
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
|
||||
from langchain.agents.agent_types import AgentType
|
||||
from langsmith import Client
|
||||
from langchain.smith import RunEvalConfig, run_on_dataset
|
||||
import pandas as pd
|
||||
from pandasai import PandasAI
|
||||
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts import ChatPromptTemplate
|
||||
from langchain.schema.output_parser import StrOutputParser
|
||||
|
||||
if __name__ == "__main__":
|
||||
df = pd.read_csv("titanic.csv")
|
||||
|
||||
pandas_ai = PandasAI(ChatOpenAI(temperature=0, model="gpt-4"), enable_cache=False)
|
||||
prompt = ChatPromptTemplate.from_messages([
|
||||
("system",
|
||||
"Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer"),
|
||||
("human", "Question: {input}"),
|
||||
("human", "Data Scientist Result: {result}"),
|
||||
])
|
||||
|
||||
def get_chain():
|
||||
chain = {
|
||||
"input": lambda x: x["input_question"],
|
||||
"result": lambda x: pandas_ai(df, prompt=x['input_question'])
|
||||
} | prompt | ChatOpenAI(temperature=0, model="gpt-4") | StrOutputParser()
|
||||
return chain
|
||||
|
||||
|
||||
client = Client()
|
||||
eval_config = RunEvalConfig(
|
||||
evaluators=[
|
||||
"qa"
|
||||
],
|
||||
)
|
||||
chain_results = run_on_dataset(
|
||||
client,
|
||||
dataset_name="Titanic CSV Data",
|
||||
llm_or_chain_factory=get_chain,
|
||||
evaluation=eval_config,
|
||||
)
|
||||
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
|
Before Width: | Height: | Size: 9.7 KiB After Width: | Height: | Size: 9.7 KiB |
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
@@ -0,0 +1,42 @@
|
||||
import pandas as pd
|
||||
import streamlit as st
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
|
||||
from langchain.agents.agent_types import AgentType
|
||||
|
||||
df = pd.read_csv('titanic.csv')
|
||||
|
||||
|
||||
llm = ChatOpenAI(temperature=0)
|
||||
agent = create_pandas_dataframe_agent(llm, df, agent_type=AgentType.OPENAI_FUNCTIONS)
|
||||
|
||||
|
||||
from langsmith import Client
|
||||
client = Client()
|
||||
def send_feedback(run_id, score):
|
||||
client.create_feedback(run_id, "user_score", score=score)
|
||||
|
||||
st.set_page_config(page_title='🦜🔗 Ask the CSV App')
|
||||
st.title('🦜🔗 Ask the CSV App')
|
||||
st.info("Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!")
|
||||
|
||||
query_text = st.text_input('Enter your question:', placeholder = 'Who was in cabin C128?')
|
||||
# Form input and query
|
||||
result = None
|
||||
with st.form('myform', clear_on_submit=True):
|
||||
submitted = st.form_submit_button('Submit')
|
||||
if submitted:
|
||||
with st.spinner('Calculating...'):
|
||||
response = agent({"input": query_text}, include_run_info=True)
|
||||
result = response["output"]
|
||||
run_id = response["__run"].run_id
|
||||
if result is not None:
|
||||
st.info(result)
|
||||
col_blank, col_text, col1, col2 = st.columns([10, 2,1,1])
|
||||
with col_text:
|
||||
st.text("Feedback:")
|
||||
with col1:
|
||||
st.button("👍", on_click=send_feedback, args=(run_id, 1))
|
||||
with col2:
|
||||
st.button("👎", on_click=send_feedback, args=(run_id, 0))
|
||||
|
||||
@@ -8,5 +8,5 @@ if __name__ == "__main__":
|
||||
output_keys=["output_text"],
|
||||
name="Titanic CSV Data",
|
||||
description="QA over titanic data",
|
||||
data_type="kv",
|
||||
data_type = "kv"
|
||||
)
|
||||
@@ -1,20 +0,0 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line, and also
|
||||
# from the environment for the first two.
|
||||
SPHINXOPTS ?=
|
||||
SPHINXBUILD ?= sphinx-build
|
||||
SOURCEDIR = source
|
||||
BUILDDIR = build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
@@ -1,35 +0,0 @@
|
||||
@ECHO OFF
|
||||
|
||||
pushd %~dp0
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=sphinx-build
|
||||
)
|
||||
set SOURCEDIR=source
|
||||
set BUILDDIR=build
|
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL
|
||||
if errorlevel 9009 (
|
||||
echo.
|
||||
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
|
||||
echo.installed, then set the SPHINXBUILD environment variable to point
|
||||
echo.to the full path of the 'sphinx-build' executable. Alternatively you
|
||||
echo.may add the Sphinx directory to PATH.
|
||||
echo.
|
||||
echo.If you don't have Sphinx installed, grab it from
|
||||
echo.https://www.sphinx-doc.org/
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
goto end
|
||||
|
||||
:help
|
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
|
||||
:end
|
||||
popd
|
||||
@@ -1,3 +0,0 @@
|
||||
chromadb/
|
||||
index.md
|
||||
Untitled.ipynb
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 18 KiB |
@@ -1,113 +0,0 @@
|
||||
# Configuration file for the Sphinx documentation builder.
|
||||
#
|
||||
# This file only contains a selection of the most common options. For a full
|
||||
# list see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
|
||||
# -- Path setup --------------------------------------------------------------
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
# import os
|
||||
# import sys
|
||||
# sys.path.insert(0, os.path.abspath('.'))
|
||||
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
import pathlib
|
||||
import sys
|
||||
from typing import List
|
||||
|
||||
import toml
|
||||
|
||||
ROOT_FOLDER = str(pathlib.Path(__file__).parent.parent.parent)
|
||||
|
||||
# Add the project root to the path
|
||||
sys.path.insert(0, ROOT_FOLDER)
|
||||
|
||||
with open("../../pyproject.toml") as f:
|
||||
data = toml.load(f)
|
||||
|
||||
project = "LangChain Benchmarks"
|
||||
copyright = "2023, Langchain AI"
|
||||
author = "Langchain AI"
|
||||
|
||||
version = data["tool"]["poetry"]["version"]
|
||||
release = version
|
||||
|
||||
html_title = project + " " + version
|
||||
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.autodoc.typehints",
|
||||
"sphinx.ext.autosummary",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.viewcode",
|
||||
"myst_nb",
|
||||
"sphinx_copybutton",
|
||||
"IPython.sphinxext.ipython_console_highlighting",
|
||||
]
|
||||
source_suffix = [".ipynb", ".html", ".md", ".rst"]
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ["_templates"]
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This pattern also affects html_static_path and html_extra_path.
|
||||
exclude_patterns: List[str] = []
|
||||
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
html_theme = "sphinx_book_theme"
|
||||
|
||||
html_theme_options = {
|
||||
"path_to_docs": "docs/source",
|
||||
"repository_url": "https://github.com/langchain-ai/langchain-benchmarks",
|
||||
"home_page_in_toc": True,
|
||||
"show_navbar_depth": 2,
|
||||
"use_sidenotes": True,
|
||||
"use_repository_button": True,
|
||||
"use_issues_button": True,
|
||||
"use_source_button": True,
|
||||
"use_fullscreen_button": True,
|
||||
"repository_branch": "main",
|
||||
"launch_buttons": {
|
||||
"notebook_interface": "jupyterlab",
|
||||
"colab_url": "https://colab.research.google.com",
|
||||
},
|
||||
}
|
||||
|
||||
html_context = {
|
||||
"display_github": True, # Integrate GitHub
|
||||
"github_user": "langchain-ai", # Username
|
||||
"github_repo": "langchain-benchmarks", # Repo name
|
||||
"github_version": "main", # Version
|
||||
"conf_py_path": "/docs/", # Path in the checkout to the docs root
|
||||
}
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ["_static"]
|
||||
|
||||
# These paths are either relative to html_static_path
|
||||
# or fully qualified paths (eg. https://...)
|
||||
html_css_files = [
|
||||
"css/custom.css",
|
||||
]
|
||||
|
||||
nb_execution_mode = "off"
|
||||
autosummary_generate = True
|
||||
@@ -1,226 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "033684fb-65b2-4586-a959-68c614741ca2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Datasets\n",
|
||||
"\n",
|
||||
"Here, we'll see how to work with LangSmith datasets."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "474292e6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install -U langchain-benchmarks"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "6d272fbf-710e-4a49-a0da-67e010541905",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks import clone_public_dataset, download_public_dataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "18ee0f96-e5c4-4ae9-aebf-7d8b88c51662",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's first download the dataset to the local file system"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "58b94f6d-0c91-4361-9b22-f758ffaa150a",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Fetching examples...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "5a2fad8c0c3549ec96a3b38fe8a002b0",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/21 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Done fetching examples.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"download_public_dataset(\n",
|
||||
" \"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/examples\"\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "841db832-b0d3-4fd1-8531-1154ec9b3caa",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"we can take a look at the first two examples"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "664e90fc-af84-4c5f-a3dd-5d9ffe649650",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[\n",
|
||||
" {\n",
|
||||
" \"created_at\": \"2023-11-15T15:26:53.511629\",\n",
|
||||
" \"dataset_id\": \"9f73165c-d333-4d14-8f59-bd7eede5db08\",\n",
|
||||
" \"id\": \"0703a989-2693-4039-a1f6-7281fc1b4cb0\",\n",
|
||||
" \"inputs\": {\n",
|
||||
" \"question\": \"do bob and alice live in the same city?\"\n",
|
||||
" },\n",
|
||||
" \"modified_at\": \"2023-11-15T15:26:53.511629\",\n",
|
||||
" \"outputs\": {\n",
|
||||
" \"expected_steps\": [\n",
|
||||
" \"find_users_by_name\",\n",
|
||||
" \"get_user_location\",\n",
|
||||
" \"get_city_for_location\",\n",
|
||||
" \"get_user_location\",\n",
|
||||
" \"get_city_for_location\"\n",
|
||||
" ],\n",
|
||||
" \"order_matters\": false,\n",
|
||||
" \"reference\": \"no\"\n",
|
||||
" },\n",
|
||||
" \"runs\": []\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"created_at\": \"2023-11-15T15:26:53.491359\",\n",
|
||||
" \"dataset_id\": \"9f73165c-d333-4d14-8f59-bd7eede5db08\",\n",
|
||||
" \"id\": \"b258b95a-9524-4da7-b758-c5481109322d\",\n",
|
||||
" \"inputs\": {\n",
|
||||
" \"question\": \"Is it likely that Donna is outside with an umbrella at this time?\"\n",
|
||||
" },\n",
|
||||
" \"modified_at\": \"2023-11-15T15:26:53.491359\",\n",
|
||||
" \"outputs\": {\n",
|
||||
" \"expected_steps\": [\n",
|
||||
" \"find_users_by_name\",\n",
|
||||
" \"get_user_location\",\n",
|
||||
" \"get_current_time_for_location\",\n",
|
||||
" \"get_current_weather_for_location\"\n",
|
||||
" ],\n",
|
||||
" \"order_matters\": false,\n",
|
||||
" \"reference\": \"yes\"\n",
|
||||
" },\n",
|
||||
" \"runs\": []\n",
|
||||
" }\n",
|
||||
"]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"with open(\"./e95d45da-aaa3-44b3-ba2b-7c15ff6e46f5.json\", \"r\", encoding=\"utf-8\") as f:\n",
|
||||
" print(json.dumps(json.load(f)[:2], indent=2, sort_keys=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2c6cf01f-466b-406d-b4c7-2395747780fd",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can also clone the dataset to our local tenant"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e4dea4df-2f1c-436b-a71c-49ffb2295ccc",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Executing this command will clone the dataset to your own LangSmith tenant. \n",
|
||||
"For this to work you must have a [LangSmith account](https://smith.langchain.com/) set up."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7eb38ea6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Get from https://smith.langchain.com/settings\n",
|
||||
"os.environ[\"LANGCHAIN_API_KEY\"] = \"ls_...\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "18d0b905-2a6a-4752-a7cb-8653bd9049e3",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"clone_public_dataset(\n",
|
||||
" \"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/examples\",\n",
|
||||
" dataset_name=\"Agent Dataset\",\n",
|
||||
")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -1,749 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e4d1cb60-6d32-4337-abee-1b6c794b7f4c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Extracting high-cardinality categoricals\n",
|
||||
"\n",
|
||||
"Suppose we built a book recommendation chatbot, and as part of it we want to extract and filter on author name if that's part of the user input. A user might ask a question like:\n",
|
||||
"\n",
|
||||
"> \"what are books about aliens by Steven King\"\n",
|
||||
"\n",
|
||||
"If we're not careful, our extraction system would most likely extract the author name \"Steven King\" from this input. This might cause us to miss all the most relevant book results, since the user was almost certainly looking for books by *Stephen King*.\n",
|
||||
"\n",
|
||||
"This is a case of having to extract a **high-cardinality categorical** value. Given a dataset of books and their respective authors, there's a large but finite number of valid author names, and we need some way of making sure our extraction system outputs valid and relevant author names even if the user input refers to invalid names. \n",
|
||||
"\n",
|
||||
"We've built a dataset to help benchmark different approaches for dealing with this challenge. The dataset is simple: it is a collection of 23 mispelled and corrected human names. To use it for high-cardinality categorical testing, we're going to generate a large set of valid names (~10,000) that includes the correct spellings of all the names in the dataset. Using this, we'll test the ability of various extraction systems to extract a corrected name from the user question:\n",
|
||||
"\n",
|
||||
"> \"what are books about aliens by {mispelled_name}\"\n",
|
||||
"\n",
|
||||
"where for each datapoint in our dataset, we'll use the mispelled name as the input and expect the corrected name as the extracted output."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "dbe58c19-c29d-41d8-844a-b03c6ee1e07a",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup\n",
|
||||
"\n",
|
||||
"We need to install a few packages and set some env vars first:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9a478941-ca99-40ee-b4f0-635f74d94a11",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install -qU langchain-benchmarks langchain-openai faker chromadb numpy scikit-learn"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8c0aa002-c334-4c51-bdf9-ffe9ae7bd56f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import getpass\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_API_KEY\"] = getpass.getpass()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "9c3dc147-2681-437e-8a26-204f10ed4d41",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from operator import attrgetter\n",
|
||||
"\n",
|
||||
"from langchain_core.prompts import ChatPromptTemplate\n",
|
||||
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
|
||||
"from langchain_core.runnables import RunnablePassthrough\n",
|
||||
"from langchain_openai import ChatOpenAI\n",
|
||||
"from langsmith import Client\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks import registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "318e0ed7-1ab5-4219-9223-900b250066de",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This is the `Name Correction` benchmark in langchain-benchmarket:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "3f2be995-b6a9-4c3d-a19f-001c0e05ac9c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"client = Client()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "cd3d005c-9b60-4bc6-a467-815e7e3bbc7c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'https://smith.langchain.com/public/78df83ee-ba7f-41c6-832c-2b23327d4cf7/d'"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task = registry[\"Name Correction\"]\n",
|
||||
"task.dataset_url"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fc4d14ea-6a46-43b1-a0ac-8e632e1297d2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"**NOTE**: If you are running this notebook for the first time, clone the public dataset into your LangSmith organization by uncommenting the below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "dca18a40-85f1-4911-9e41-936975fbddf8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# client.clone_public_dataset(task.dataset_url)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "3f9ad08e-69cc-436e-94f9-b0e1e2c4a9d1",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'name': 'Tracy Cook'} {'name': 'Traci Cook'}\n",
|
||||
"{'name': 'Dan Klein'} {'name': 'Daniel Klein'}\n",
|
||||
"{'name': 'Jen Mcintosh'} {'name': 'Jennifer Mcintosh'}\n",
|
||||
"{'name': 'Cassie Hull'} {'name': 'Cassandra Hull'}\n",
|
||||
"{'name': 'Andy Williams'} {'name': 'Andrew Williams'}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"examples = list(client.list_examples(dataset_name=task.dataset_name))\n",
|
||||
"for example in examples[:5]:\n",
|
||||
" print(example.inputs, example.outputs)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"id": "35c85a6f-5d8d-4018-9b83-b6cab0587c1c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def run_on_dataset(chain, run_name):\n",
|
||||
" client.run_on_dataset(\n",
|
||||
" dataset_name=task.dataset_name,\n",
|
||||
" llm_or_chain_factory=chain,\n",
|
||||
" evaluation=task.eval_config,\n",
|
||||
" project_name=run_name,\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4fd7318a-4195-4da8-94d7-34ee6b7c2097",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Augmenting with more fake names\n",
|
||||
"\n",
|
||||
"For our tests we'll create a list of 10,000 names that represent all the possible values for this category. This will include our target names from the dataset."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "06098983-f5cf-4de3-ae07-4cdbe091522c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from faker import Faker\n",
|
||||
"\n",
|
||||
"Faker.seed(42)\n",
|
||||
"fake = Faker()\n",
|
||||
"fake.seed_instance(0)\n",
|
||||
"\n",
|
||||
"incorrect_names = [example.inputs[\"name\"] for example in examples]\n",
|
||||
"correct_names = [example.outputs[\"name\"] for example in examples]\n",
|
||||
"\n",
|
||||
"# We'll make sure that our list of valid names contains the correct spellings\n",
|
||||
"# and not the incorrect spellings from our dataset\n",
|
||||
"valid_names = list(\n",
|
||||
" set([fake.name() for _ in range(10_000)] + correct_names).difference(\n",
|
||||
" incorrect_names\n",
|
||||
" )\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "ab6d9b4b-717b-4947-ac17-a100a0ced088",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"9382"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"len(valid_names)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "6e7d27bf-c82c-43e1-961a-ea67733b1dec",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['Debra Lee', 'Kevin Harper', 'Donald Anderson']"
|
||||
]
|
||||
},
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"valid_names[:3]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bd801ab5-b2a4-49bc-9c11-698dc760eb28",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Chain 1: Baseline\n",
|
||||
"\n",
|
||||
"As a baseline we'll create a function-calling chain that has no information about the set of valid names."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "1e0694d9-d67d-4f90-b40c-f8373389f5c4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class Search(BaseModel):\n",
|
||||
" query: str\n",
|
||||
" author: str\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"system = \"\"\"Generate a relevant search query for a library system\"\"\"\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", \"{system}\"),\n",
|
||||
" (\"human\", \"what are books about aliens by {name}\"),\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
|
||||
"structured_llm = llm.with_structured_output(Search)\n",
|
||||
"\n",
|
||||
"query_analyzer_1 = (\n",
|
||||
" prompt.partial(system=system) | structured_llm | {\"name\": attrgetter(\"author\")}\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "f4a4d81f-532a-4efb-86cb-cc0555dbc4e7",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'GPT-3.5' at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6/compare?selectedSessions=f429ec84-b879-4e66-b7fb-ef7be69d1acd\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Extracting Corrected Names at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6\n",
|
||||
"[------------------------------------------------->] 23/23"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"run_on_dataset(query_analyzer_1, \"GPT-3.5\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b4f42968-069f-450b-a03b-f47934956f89",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"As we might have expected, this gives us a `Correct rate: 0%`. Let's see if we can do better :)\n",
|
||||
"\n",
|
||||
"See the test run in LangSmith [here](https://smith.langchain.com/public/8c0a4c25-426d-4582-96fc-d7def170be76/d/compare?selectedSessions=f429ec84-b879-4e66-b7fb-ef7be69d1acd)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "08ef2fc6-0ad9-4a3e-a306-bd7100f7b1fb",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Chain 2: All candidates in prompt\n",
|
||||
"\n",
|
||||
"Next, let's dump the full list of valid names in the system prompt. We'll need a model with a longer context window than the 16k token window of gpt-3.5-turbo-0125 so we'll use gpt-4-0125-preview."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "d0f65f4f-5461-43b1-9c7b-5fcdaf48c2ce",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"valid_names_str = \"\\n\".join(valid_names)\n",
|
||||
"\n",
|
||||
"system_2 = \"\"\"Generate a relevant search query for a library system.\n",
|
||||
"\n",
|
||||
"`author` attribute MUST be one of:\n",
|
||||
"\n",
|
||||
"{valid_names_str}\n",
|
||||
"\n",
|
||||
"Do NOT hallucinate author name!\"\"\"\n",
|
||||
"\n",
|
||||
"formatted_system = system_2.format(valid_names_str=valid_names_str)\n",
|
||||
"structured_llm_2 = ChatOpenAI(\n",
|
||||
" model=\"gpt-4-0125-preview\", temperature=0\n",
|
||||
").with_structured_output(Search)\n",
|
||||
"query_analyzer_2 = (\n",
|
||||
" prompt.partial(system=formatted_system)\n",
|
||||
" | structured_llm_2\n",
|
||||
" | {\"name\": attrgetter(\"author\")}\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "de679906-c69d-4ceb-bc5e-73a291b21cdc",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'GPT-4, all names in prompt' at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6/compare?selectedSessions=8c4cfdfc-3646-438e-be47-43a40d66292a\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Extracting Corrected Names at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6\n",
|
||||
"[------------------------------------------------->] 23/23"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"run_on_dataset(query_analyzer_2, \"GPT-4, all names in prompt\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eb678fdd-0e57-4063-adea-56248aea11e5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This gets us up to `Correct rate: 26%`.\n",
|
||||
"\n",
|
||||
"See the test run in LangSmith [here](https://smith.langchain.com/public/8c0a4c25-426d-4582-96fc-d7def170be76/d/compare?selectedSessions=8c4cfdfc-3646-438e-be47-43a40d66292a)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0aa394b5-a665-4f4c-809d-c0d756c9b23e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Chain 3: Top k candidates from vectorstore in prompt\n",
|
||||
"\n",
|
||||
"10,000 names is a lot to have in the prompt. Perhaps we could get better performance by shortening the list using vector search first to only include names that have the highest similarity to the user question. We can return to using GPT-3.5 as a result:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f9439e3f-5aa2-45b7-ab1f-149060744e03",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_community.vectorstores import Chroma\n",
|
||||
"from langchain_core.prompts import PromptTemplate\n",
|
||||
"from langchain_openai import OpenAIEmbeddings\n",
|
||||
"\n",
|
||||
"k = 10\n",
|
||||
"embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n",
|
||||
"vectorstore = Chroma.from_texts(valid_names, embeddings, collection_name=\"author_names\")\n",
|
||||
"retriever = vectorstore.as_retriever(search_kwargs={\"k\": k})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"id": "04018b30-2378-4c96-8515-39d66c554459",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"system_chain = (\n",
|
||||
" (lambda name: f\"what are books about aliens by {name}\")\n",
|
||||
" | retriever\n",
|
||||
" | (\n",
|
||||
" lambda docs: system_2.format(\n",
|
||||
" valid_names_str=\"\\n\".join(d.page_content for d in docs)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"query_analyzer_3 = (\n",
|
||||
" RunnablePassthrough.assign(system=system_chain)\n",
|
||||
" | prompt\n",
|
||||
" | structured_llm\n",
|
||||
" | {\"name\": attrgetter(\"author\")}\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"id": "fd5af75e-41fa-42ee-b9ac-62eb13e21022",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'GPT-3.5, top 10 names in prompt, vecstore' at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6/compare?selectedSessions=af93ec50-ccbb-4b3c-908a-70c75e5516ea\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Extracting Corrected Names at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6\n",
|
||||
"[------------------------------------------------->] 23/23"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"run_on_dataset(query_analyzer_3, f\"GPT-3.5, top {k} names in prompt, vecstore\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b7e0f097-7432-4728-a60b-b980046c1275",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This gets us up to `Correct rate: 57%`\n",
|
||||
"\n",
|
||||
"See the test run in LangSmith [here](https://smith.langchain.com/public/8c0a4c25-426d-4582-96fc-d7def170be76/d/compare?selectedSessions=af93ec50-ccbb-4b3c-908a-70c75e5516ea)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "20aaa33a-d475-41a1-8f1a-53e18382b3d7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Chain 4: Top k candidates by ngram overlap in prompt\n",
|
||||
"\n",
|
||||
"Instead of using vector search, which requires embeddings and vector stores, a cheaper and faster approach would be to compare ngram overlap between the user question and the list of valid names:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"id": "05b2fc1c-0f61-4638-bbf5-fed5b634db51",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
||||
"from sklearn.metrics.pairwise import cosine_similarity\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Function to generate character n-grams\n",
|
||||
"def ngrams(string, n=3):\n",
|
||||
" string = \"START\" + string.replace(\" \", \"\").lower() + \"END\"\n",
|
||||
" ngrams = zip(*[string[i:] for i in range(n)])\n",
|
||||
" return [\"\".join(ngram) for ngram in ngrams]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Vectorize documents using TfidfVectorizer with the custom n-grams function\n",
|
||||
"vectorizer = TfidfVectorizer(analyzer=ngrams)\n",
|
||||
"tfidf_matrix = vectorizer.fit_transform(valid_names)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"id": "2994aff8-4bfd-4cf3-9b73-2bda7c470ba4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_names(query):\n",
|
||||
" # Vectorize query\n",
|
||||
" query_tfidf = vectorizer.transform([query])\n",
|
||||
"\n",
|
||||
" # Compute cosine similarity\n",
|
||||
" cosine_similarities = cosine_similarity(query_tfidf, tfidf_matrix).flatten()\n",
|
||||
"\n",
|
||||
" # Find the index of the most similar document\n",
|
||||
" most_similar_document_indexes = np.argsort(-cosine_similarities)\n",
|
||||
"\n",
|
||||
" return \"\\n\".join([valid_names[i] for i in most_similar_document_indexes[:k]])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"id": "a549a347-1449-4ae2-a30d-e8f0b917d50e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_system_prompt(input):\n",
|
||||
" name = input[\"name\"]\n",
|
||||
" valid_names_str = get_names(f\"what are books about aliens by {name}\")\n",
|
||||
" return system_2.format(valid_names_str=valid_names_str)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"query_analyzer_4 = (\n",
|
||||
" RunnablePassthrough.assign(system=get_system_prompt)\n",
|
||||
" | prompt\n",
|
||||
" | structured_llm\n",
|
||||
" | {\"name\": attrgetter(\"author\")}\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"id": "dd1b69a8-5ca6-4a2d-9ad3-567d0105b672",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'GPT-3.5, top 10 names in prompt, ngram' at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6/compare?selectedSessions=bc28b761-2ac9-4391-8df1-758f0a4d5100\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Extracting Corrected Names at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6\n",
|
||||
"[------------------------------------------------->] 23/23"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"run_on_dataset(query_analyzer_4, f\"GPT-3.5, top {k} names in prompt, ngram\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b4e16c1b-33d5-4ca1-932b-8234ffc668bf",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This gets us up to `Correct rate: 65%`\n",
|
||||
"\n",
|
||||
"See the test run in LangSmith [here](https://smith.langchain.com/public/8c0a4c25-426d-4582-96fc-d7def170be76/d/compare?selectedSessions=bc28b761-2ac9-4391-8df1-758f0a4d5100)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d3045376-e102-4ec6-877a-91448677f3f3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Chain 5: Replace with top candidate from vectorstore\n",
|
||||
"\n",
|
||||
"Instead of (or in addition to) searching for similar candidates before extraction, we can also compare and correct the extracted value after-the-fact a search over the valid names. With Pydantic classes this is easy using a validator:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 28,
|
||||
"id": "ac719651-0775-4fa4-bd22-9fddebcc6918",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_core.pydantic_v1 import validator\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class Search(BaseModel):\n",
|
||||
" query: str\n",
|
||||
" author: str\n",
|
||||
"\n",
|
||||
" @validator(\"author\")\n",
|
||||
" def double(cls, v: str) -> str:\n",
|
||||
" return vectorstore.similarity_search(v, k=1)[0].page_content\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"structured_llm_3 = llm.with_structured_output(Search)\n",
|
||||
"query_analyzer_5 = (\n",
|
||||
" prompt.partial(system=system) | structured_llm_3 | {\"name\": attrgetter(\"author\")}\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"id": "fc1cfdcb-47fb-40c4-898d-f290cd53a37d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'GPT-3.5, correct name, vecstore' at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6/compare?selectedSessions=e3eda1e1-bc25-46e8-a4fb-db324cefd1c9\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Extracting Corrected Names at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6\n",
|
||||
"[------------------------------------------------->] 23/23"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"run_on_dataset(query_analyzer_5, f\"GPT-3.5, correct name, vecstore\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e6e96a2c-506e-461f-bd05-cb88fe0ea3aa",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This gets us up to `Correct rate: 83%`\n",
|
||||
"\n",
|
||||
"See the test run in LangSmith [here](https://smith.langchain.com/public/8c0a4c25-426d-4582-96fc-d7def170be76/d/compare?selectedSessions=e3eda1e1-bc25-46e8-a4fb-db324cefd1c9)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f1f8ce77-01a3-41d1-a047-103cb2e552f9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Chain 6: Replace with top candidate by ngram overlap\n",
|
||||
"\n",
|
||||
"We can do the same with ngram overlap search instead of vector search:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"id": "21ffa8c9-907b-453a-9b32-01a981bca5ec",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class Search(BaseModel):\n",
|
||||
" query: str\n",
|
||||
" author: str\n",
|
||||
"\n",
|
||||
" @validator(\"author\")\n",
|
||||
" def double(cls, v: str) -> str:\n",
|
||||
" return get_names(v).split(\"\\n\")[0]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"structured_llm_4 = llm.with_structured_output(Search)\n",
|
||||
"query_analyzer_6 = (\n",
|
||||
" prompt.partial(system=system) | structured_llm_4 | {\"name\": attrgetter(\"author\")}\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 31,
|
||||
"id": "126354dd-c54e-4391-8a5e-5e200d006a18",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'GPT-3.5, correct name, ngram' at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6/compare?selectedSessions=8f8846c8-2ada-41bc-8d2c-e1d56e7c92ce\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Extracting Corrected Names at:\n",
|
||||
"https://smith.langchain.com/o/43ae1439-dbb7-53b8-bef4-155154d3f962/datasets/1765d6b2-aa2e-46ec-9158-9f4ca8f228c6\n",
|
||||
"[------------------------------------------------->] 23/23"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"run_on_dataset(query_analyzer_6, f\"GPT-3.5, correct name, ngram\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b8c8cd81-61d0-4c1f-957d-1910be7706e7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This gets us up to `Correct rate: 74%`, slightly worse than Chain 5 (same thing using vector search insteadf of ngram).\n",
|
||||
"\n",
|
||||
"See the test run in LangSmith [here](https://smith.langchain.com/public/8c0a4c25-426d-4582-96fc-d7def170be76/d/compare?selectedSessions=8f8846c8-2ada-41bc-8d2c-e1d56e7c92ce)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4d7f7ab4-466d-434c-98bd-ebe1906599a9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## See all results in LangSmith\n",
|
||||
"\n",
|
||||
"To see the full dataset and all the test results, head to LangSmith: https://smith.langchain.com/public/8c0a4c25-426d-4582-96fc-d7def170be76/d"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "benchmarks-venv",
|
||||
"language": "python",
|
||||
"name": "benchmarks-venv"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7e8fc49a-e8b2-404b-a059-e9f668c460e5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Introduction\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"These tasks refer to an LLM's ability to extract structured output from an unstructured source, such as emails, websites, or other text. Below are a list of supported datasets.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "86912590-a90a-4351-8ab4-89192cdee1e7",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<thead>\n",
|
||||
"<tr><th>Name </th><th>Type </th><th>Dataset ID </th><th>Description </th></tr>\n",
|
||||
"</thead>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Email Extraction</td><td>ExtractionTask</td><td><a href=\"https://smith.langchain.com/public/a1742786-bde5-4f51-a1d8-e148e5251ddb/d\" target=\"_blank\" rel=\"noopener\">a1742786-bde5-4f51-a1d8-e148e5251ddb</a></td><td>A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\n",
|
||||
"\n",
|
||||
"Some additional cleanup of the data was done by hand after the initial pass.\n",
|
||||
"\n",
|
||||
"See https://github.com/jacoblee93/oss-model-extraction-evals. </td></tr>\n",
|
||||
"<tr><td>Chat Extraction </td><td>ExtractionTask</td><td><a href=\"https://smith.langchain.com/public/00f4444c-9460-4a82-b87a-f50096f1cfef/d\" target=\"_blank\" rel=\"noopener\">00f4444c-9460-4a82-b87a-f50096f1cfef</a></td><td>A dataset meant to test the ability of an LLM to extract and infer\n",
|
||||
"structured information from a dialogue. The dialogue is between a user and a support\n",
|
||||
"engineer. Outputs should be structured as a JSON object and test both the ability\n",
|
||||
"of the LLM to correctly structure the information and its ability to perform simple \n",
|
||||
"classification tasks. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"Registry(tasks=[ExtractionTask(name='Email Extraction', dataset_id='https://smith.langchain.com/public/a1742786-bde5-4f51-a1d8-e148e5251ddb/d', description='A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\\n\\nSome additional cleanup of the data was done by hand after the initial pass.\\n\\nSee https://github.com/jacoblee93/oss-model-extraction-evals.\\n ', schema=<class 'langchain_benchmarks.extraction.tasks.email_task.Email'>, instructions=ChatPromptTemplate(input_variables=['input'], messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are an expert researcher.')), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['input'], template='What can you tell me about the following email? Make sure to extract the question in the correct format. Here is the email:\\n ```\\n{input}\\n```'))])), ExtractionTask(name='Chat Extraction', dataset_id='https://smith.langchain.com/public/00f4444c-9460-4a82-b87a-f50096f1cfef/d', description='A dataset meant to test the ability of an LLM to extract and infer\\nstructured information from a dialogue. The dialogue is between a user and a support\\nengineer. Outputs should be structured as a JSON object and test both the ability\\nof the LLM to correctly structure the information and its ability to perform simple \\nclassification tasks.', schema=<class 'langchain_benchmarks.extraction.tasks.chat_extraction.schema.GenerateTicket'>, instructions=ChatPromptTemplate(input_variables=['dialogue'], messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are a helpdesk assistant responsible with extracting information and generating tickets. Dialogues are between a user and a support engineer.')), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['dialogue'], template='Generate a ticket for the following question-response pair:\\n<Dialogue>\\n{dialogue}\\n</Dialogue>'))]))])"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_benchmarks import registry\n",
|
||||
"\n",
|
||||
"registry.filter(Type=\"ExtractionTask\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e771e544-b8f5-4359-8fd7-c89b71fbe460",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Task resources\n",
|
||||
"\n",
|
||||
"In addition to the dataset_id, name, and description, each extraction task provides the following:\n",
|
||||
"\n",
|
||||
"- `schema` - a pydantic base model defining the schema (or schemas) the model should extract\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Dataset schema\n",
|
||||
"\n",
|
||||
"Each task corresponds to a LangSmith dataset with the following schema:\n",
|
||||
"\n",
|
||||
"Inputs:\n",
|
||||
"- `input: str` - the input text\n",
|
||||
"\n",
|
||||
"Outputs\n",
|
||||
"- `output: str` - the expected extraction result, as a json object\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d04e05f3-3f20-4fed-bb98-3eb072213bbd",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Evaluation\n",
|
||||
"\n",
|
||||
"The extraction tasks also have an evaluation config, which defines default LangSmith evaluators to apply when benchmarking your architecture.\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "9c7865bd-8251-4579-85a3-f9085d96f497",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.extraction import get_eval_config\n",
|
||||
"\n",
|
||||
"eval_llm = ChatOpenAI(model=\"gpt-4\", model_kwargs={\"seed\": 42})\n",
|
||||
"eval_config = get_eval_config(eval_llm)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,946 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e2af633b-4d0f-4b80-b090-2d6429f22e90",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Evaluating RAG Architectures on Benchmark Tasks\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"#### Introduction\n",
|
||||
"\n",
|
||||
"If you ever wanted to compare different approaches to Q&A over docs, you'll find this notebook helpful to get started evaluating different configurations and common RAG architectures on benchmark tasks. The goal is to make it easy for you to experiment with different techniques, understand their tradeoffs, and make informed decisions for your specific use case.\n",
|
||||
"\n",
|
||||
"#### What is RAG?\n",
|
||||
"\n",
|
||||
"LLMs have a knowledge cutoff. For them to accurately respond to user queries, they need access to relevant information. Retrieval Augmented Generation (RAG) (aka \"give an LLM a search engine\") is a common design pattern to address this. The key components are:\n",
|
||||
"\n",
|
||||
"- Retriever: fetches information from a knowledge base, which can be a vector search engine, a database, or any search engine.\n",
|
||||
"- Generator: synthesizes responses using a blend of learned knowledge and the retrieved information.\n",
|
||||
"\n",
|
||||
"The overall quality of the system depends on both components.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"#### Benchmark Tasks and Datasets (As of 2023/11/21)\n",
|
||||
"\n",
|
||||
"The following datasets are currently available:\n",
|
||||
"\n",
|
||||
"- LangChain Docs Q&A - technical questions based on the LangChain python documentation\n",
|
||||
"- Semi-structured Earnings - financial questions and answers on financial PDFs containing tables and graphs\n",
|
||||
"\n",
|
||||
"Each task comes with a labeled dataset of questions and answers. They also provide configurable factory functions for easy customization of chunking and indexing for the relevant source documents.\n",
|
||||
"\n",
|
||||
"And with that, let's get started!\n",
|
||||
"\n",
|
||||
"## Pre-requisites\n",
|
||||
"\n",
|
||||
"We will install quite a few prerequisites for this example since we are comparing many techniques and models.\n",
|
||||
"\n",
|
||||
"We will be using LangSmith to capture the evaluation traces. You can make a free account at [smith.langchain.com](https://smith.langchain.com/). Once you've done so, you can make an API key and set it below.\n",
|
||||
"\n",
|
||||
"We are comparing many methods throughout this notebook, so the list of dependencies we will install is long."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9f44b59b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install -U --quiet langchain langsmith langchainhub langchain_benchmarks\n",
|
||||
"%pip install --quiet chromadb openai huggingface pandas langchain_experimental sentence_transformers pyarrow anthropic tiktoken"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "62b518cf-99fb-44be-8acb-ee0a8ba62272",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
|
||||
"os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\" # Your API key\n",
|
||||
"os.environ[\"OPENAI_API_KEY\"] = \"sk-...\" # Your OpenAI API key\n",
|
||||
"os.environ[\"ANTHROPIC_API_KEY\"] = \"sk-...\" # Your Anthropic API key\n",
|
||||
"# Silence warnings from HuggingFace\n",
|
||||
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"# Generate a unique run ID for these experiments\n",
|
||||
"run_uid = uuid.uuid4().hex[:6]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2e8a666d-8bf5-4bfd-8b20-8b7defdb8cd5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Review Q&A tasks\n",
|
||||
"\n",
|
||||
"The registry provides configurations to test out common architectures on curated datasets.\n",
|
||||
"Below is a list of the available tasks at the time of writing."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks import clone_public_dataset, registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "3644d211-382e-41aa-b282-21b01d28fc35",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<thead>\n",
|
||||
"<tr><th>Name </th><th>Type </th><th>Dataset ID </th><th>Description </th></tr>\n",
|
||||
"</thead>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>LangChain Docs Q&A </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
|
||||
"\n",
|
||||
"The environment provides the documents and the retriever information.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"<tr><td>Semi-structured Reports</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
|
||||
"\n",
|
||||
"The task provides the raw documents as well as factory methods to easily index them\n",
|
||||
"and create a retriever.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x12aae2840>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x12aae28e0>, 'hyde': <function _chroma_hyde_retriever_factory at 0x12aae2980>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x12a1be020>}, get_docs=<function load_cached_docs at 0x12a1bdb20>), RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x12aae3060>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x12aae3100>, 'hyde': <function _chroma_hyde_retriever_factory at 0x12aae31a0>}, architecture_factories={}, get_docs=<function load_docs at 0x12aae2fc0>)])"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"registry.filter(Type=\"RetrievalTask\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "671282f8-c455-4390-b018-e53bbd833093",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"langchain_docs = registry[\"LangChain Docs Q&A\"]\n",
|
||||
"langchain_docs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "70369f67-deb4-467a-801a-6d38c3d0460d",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"clone_public_dataset(langchain_docs.dataset_id, dataset_name=langchain_docs.name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "02011398-1a6f-42c1-b586-9d01c78e3ee4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Basic Vector Retrieval\n",
|
||||
"\n",
|
||||
"For our first example, we will generate a single embedding for each document in the dataset,\n",
|
||||
"without chunking or indexing, and then provide that retriever to an LLM for inference."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c58247f5-b9bd-4cc5-9632-78bc21bb10b4",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.embeddings import HuggingFaceEmbeddings\n",
|
||||
"\n",
|
||||
"embeddings = HuggingFaceEmbeddings(\n",
|
||||
" model_name=\"thenlper/gte-base\",\n",
|
||||
" model_kwargs={\"device\": 0}, # Comment out to use CPU\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"retriever_factory = langchain_docs.retriever_factories[\"basic\"]\n",
|
||||
"# Indexes the documents with the specified embeddings\n",
|
||||
"# Note that this does not apply any chunking to the docs,\n",
|
||||
"# which means the documents can be of arbitrary length\n",
|
||||
"retriever = retriever_factory(embeddings)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f4d2e139-2653-4f7b-944b-91ef52f43d3e",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Factory for creating a conversational retrieval QA chain\n",
|
||||
"\n",
|
||||
"chain_factory = langchain_docs.architecture_factories[\"conversational-retrieval-qa\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9f9be718-64f0-4706-9527-240a1cdb3ecb",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatAnthropic\n",
|
||||
"\n",
|
||||
"# Example\n",
|
||||
"llm = ChatAnthropic(model=\"claude-2\", temperature=1)\n",
|
||||
"\n",
|
||||
"chain_factory(retriever, llm=llm).invoke({\"question\": \"what's lcel?\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "513042fe-2878-44f8-ae84-05b9d521c1de",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from functools import partial\n",
|
||||
"\n",
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.rag import get_eval_config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "aab7514e-a6ef-4c21-b90f-d9cbefcf5af1",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"client = Client()\n",
|
||||
"RAG_EVALUATION = get_eval_config()\n",
|
||||
"\n",
|
||||
"test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=partial(chain_factory, retriever, llm=llm),\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"claude-2 qa-chain simple-index {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"basic\",\n",
|
||||
" \"embedding_model\": \"thenlper/gte-base\",\n",
|
||||
" \"llm\": \"claude-2\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e86578d5-be5c-4bcd-9dcb-35280eeed3f9",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_run.get_aggregate_feedback()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ee992f87-4137-49b1-a1f1-0cc7be0e32d8",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Comparing with other indexing strategies\n",
|
||||
"\n",
|
||||
"The index used above retrieves the raw documents based on a single vector per document. It doesn't perform any additional chunking. You can try changing the chunking parameters when generating the index.\n",
|
||||
"\n",
|
||||
"## Customizing Chunking\n",
|
||||
"\n",
|
||||
"The simplest change you can make to the index is configure how you split the documents."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e72030d4-c201-44b8-85cd-903afa313f11",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def transform_docs(docs):\n",
|
||||
" splitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=200)\n",
|
||||
" yield from splitter.split_documents(docs)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Used for the cache\n",
|
||||
"transformation_name = \"recursive-text-cs4k-ol200\"\n",
|
||||
"\n",
|
||||
"retriever_factory = langchain_docs.retriever_factories[\"basic\"]\n",
|
||||
"\n",
|
||||
"chunked_retriever = retriever_factory(\n",
|
||||
" embeddings,\n",
|
||||
" transform_docs=transform_docs,\n",
|
||||
" transformation_name=transformation_name,\n",
|
||||
" search_kwargs={\"k\": 4},\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d74f12f9-1ba6-4bf7-a850-4073fb0994f9",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"chunked_results = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=partial(chain_factory, chunked_retriever, llm=llm),\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"claude-2 qa-chain chunked {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"basic\",\n",
|
||||
" \"chunk_size\": 4000,\n",
|
||||
" \"chunk_overlap\": 200,\n",
|
||||
" \"embedding_model\": \"thenlper/gte-base\",\n",
|
||||
" \"llm\": \"claude-2\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d6d825f1-9a91-429d-bf3e-a9b9c2785a69",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"chunked_results.get_aggregate_feedback()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5a7a62ec-9a9c-4d7a-ab90-97020d855ee7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Parent Document Retriever\n",
|
||||
"\n",
|
||||
"This indexing technique chunks documents and generates 1 vector per chunk.\n",
|
||||
"At retrieval time, the K \"most similar\" chunks are fetched, then the full parent documents are returned for the LLM to reason over.\n",
|
||||
"\n",
|
||||
"This ensures the chunk is surfaced in its full natural context. It also can potentially improve the initial retrieval quality since the similarity scores are scoped to individual chunks.\n",
|
||||
"\n",
|
||||
"Let's see if this technique is effective in our case."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1398f5e3-b7fe-4693-bcc0-c6c6f75c8234",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"retriever_factory = langchain_docs.retriever_factories[\"parent-doc\"]\n",
|
||||
"\n",
|
||||
"# Indexes the documents with the specified embeddings\n",
|
||||
"parent_doc_retriever = retriever_factory(embeddings)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7b1f4b5d-143a-44ce-95f4-d0b5782ada74",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"parent_doc_test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=partial(chain_factory, parent_doc_retriever, llm=llm),\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"claude-2 qa-chain parent-doc {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"parent-doc\",\n",
|
||||
" \"embedding_model\": \"thenlper/gte-base\",\n",
|
||||
" \"llm\": \"claude-2\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3cef0410-47ec-4830-9b75-621eb85240ed",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"parent_doc_test_run.get_aggregate_feedback()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d4b27dd0-f0df-4551-a972-1a6c0df5ffb9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## HyDE\n",
|
||||
"\n",
|
||||
"HyDE (Hypothetical document embeddings) refers to the technique of using an LLM\n",
|
||||
"to generate example queries that my be used to retrieve a doc. By doing so, the resulting embeddings are automatically \"more aligned\" with the embeddings generated from the query. This comes with an additional indexing cost, since each document requires an additoinal call to an LLM while indexing."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9c92d2c2-f410-43cc-9c9f-abc22ef48353",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"retriever_factory = langchain_docs.retriever_factories[\"hyde\"]\n",
|
||||
"\n",
|
||||
"retriever = retriever_factory(embeddings)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2179cf29-2d75-4a04-bbb5-b8f22028fa34",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"hyde_test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=partial(chain_factory, retriever=retriever, llm=llm),\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" verbose=True,\n",
|
||||
" project_name=f\"claude-2 qa-chain HyDE {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"HyDE\",\n",
|
||||
" \"embedding_model\": \"thenlper/gte-base\",\n",
|
||||
" \"llm\": \"claude-2\",\n",
|
||||
" },\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "94a04f21-0308-4b00-a6f1-694d98ba7109",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"hyde_test_run.get_aggregate_feedback()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2c8af309-d0c4-4562-a5f0-30ca9f9fd861",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Comparing Embeddings\n",
|
||||
"\n",
|
||||
"We've been using off-the-shelf GTE-Base embeddings so far to retrieve the docs, but\n",
|
||||
"you may get better results with other embeddings. You could even try fine-tuning embedddings on your own documentation and evaluating here.\n",
|
||||
"\n",
|
||||
"Let's compare our results so far to OpenAI's embeddings."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4e0b2395-c07e-4eae-bb21-afdda3961cc2",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.embeddings.openai import OpenAIEmbeddings\n",
|
||||
"\n",
|
||||
"openai_embeddings = OpenAIEmbeddings()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "14a5edab-9a3a-4864-b69f-69bc1c9e7816",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"openai_retriever = langchain_docs.retriever_factories[\"basic\"](openai_embeddings)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6757c411-aaa5-42ad-824c-7c0b5b942e40",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"openai_embeddings_test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=partial(chain_factory, openai_retriever),\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"claude-2 qa-chain oai-emb basic {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"basic\",\n",
|
||||
" \"embedding_model\": \"openai/text-embedding-ada-002\",\n",
|
||||
" \"llm\": \"claude-2\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a8ae7cbe-a8eb-4b40-aeae-f9c7f4bf335f",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"openai_embeddings_test_run.get_aggregate_feedback()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3ef164b9-7124-4907-b2b4-0595bf3b3441",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Comparing Models\n",
|
||||
"\n",
|
||||
"We used Anthropic's Claude-2 model in our previous tests, but lets try with some other models.\n",
|
||||
"\n",
|
||||
"You can swap in any LangChain LLM within the response generator below.\n",
|
||||
"We'll try a long-context llama 2 model first (using Ollama)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "402c86c7-9754-4527-a1a9-a89beba437b4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatOllama\n",
|
||||
"\n",
|
||||
"# A llama2-based model with 128k context\n",
|
||||
"# (in theory) In practice, we will see how well\n",
|
||||
"# it actually leverages that context.\n",
|
||||
"ollama = ChatOllama(model=\"yarn-llama2:7b-128k\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fc7dff86-2b93-490a-81ab-72e757e8f1b3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# We'll go back to the GTE embeddings for now\n",
|
||||
"\n",
|
||||
"retriever_factory = langchain_docs.retriever_factories[\"basic\"]\n",
|
||||
"retriever = retriever_factory(embeddings)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "eaa47085-e383-4cc5-9018-5491700c6f71",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ollama_test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=partial(chain_factory, llm=ollama, retriever=retriever),\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"yarn-llama2:7b-128k qa-chain basic {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"basic\",\n",
|
||||
" \"embedding_model\": \"thenlper/gte-base\",\n",
|
||||
" \"llm\": \"ollama/yarn-llama2:7b-128k\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "98edcf42-a405-400b-882e-04de2559359c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Changing the prompt in the response generator\n",
|
||||
"\n",
|
||||
"The default prompt was tested primariily on OpenAI's gpt-3.5 model. When switching models, you may get better results if you modify the prompt. Let's try a simple one."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "69d3b36f-68aa-4005-9bb2-de228491ef86",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain import hub\n",
|
||||
"from langchain.schema.output_parser import StrOutputParser"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "64caac6f-888d-432c-9329-5c4b97ad859d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prompt = hub.pull(\"wfh/rag-simple\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3c0e6762-1e50-4eef-833a-a4a2bf8883ba",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"generator = prompt | ChatAnthropic(model=\"claude-2\", temperature=1) | StrOutputParser()\n",
|
||||
"new_chain = chain_factory(response_generator=generator, retriever=openai_retriever)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "96886de0-a653-4875-a68f-5a11efcb200b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"claude_simple_prompt_test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=partial(\n",
|
||||
" chain_factory, response_generator=generator, retriever=retriever, llm=llm\n",
|
||||
" ),\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"claude-2 qa-chain basic rag-simple {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"basic\",\n",
|
||||
" \"embedding_model\": \"thenlper/gte-base\",\n",
|
||||
" \"prompt\": \"wfh/rag-simple\",\n",
|
||||
" \"llm\": \"claude-2\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "daffaf28-902e-4466-b3b9-25441d45585d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Testing Agents\n",
|
||||
"\n",
|
||||
"Agents use an LLM to decide actions and generate responses. There are two obvious ways they could potentially succeed where the approaches above fail:\n",
|
||||
"- The above chains do not \"rephrase\" the user query. It could be that the rephrased question will result in more relevant documents.\n",
|
||||
"- The above chains must respond based on a single retrieval step. Agents can iteratively query the retriever or subdivide the query into different parts to synthesize at the end. Our dataset has a number of questions that require information from different documents - if the\n",
|
||||
"\n",
|
||||
"Let's evaluate to see whether the \"plausible\" statements above are worth the tradeoffs. We will use the basic retriever as a tool for them."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c31c19c4-f8d6-41b3-9389-e89abd4b5f04",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import List, Tuple\n",
|
||||
"\n",
|
||||
"from langchain.agents import AgentExecutor\n",
|
||||
"from langchain.agents.format_scratchpad import format_to_openai_functions\n",
|
||||
"from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n",
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
|
||||
"from langchain.pydantic_v1 import BaseModel, Field\n",
|
||||
"from langchain.schema.messages import AIMessage, HumanMessage\n",
|
||||
"from langchain.tools import tool\n",
|
||||
"from langchain.tools.render import format_tool_to_openai_function\n",
|
||||
"\n",
|
||||
"# This is used to tell the model how to best use the retriever.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"@tool\n",
|
||||
"def search(query, callbacks=None):\n",
|
||||
" \"\"\"Search the LangChain docs with the retriever.\"\"\"\n",
|
||||
" return retriever.get_relevant_documents(query, callbacks=callbacks)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tools = [search]\n",
|
||||
"\n",
|
||||
"llm = ChatOpenAI(model=\"gpt-4-1106-preview\", temperature=0)\n",
|
||||
"assistant_system_message = \"\"\"You are a helpful assistant tasked with answering technical questions about LangChain. \\\n",
|
||||
"Use tools (only if necessary) to best answer the users questions. Do not make up information if you cannot find the answer using your tools.\"\"\"\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", assistant_system_message),\n",
|
||||
" MessagesPlaceholder(variable_name=\"chat_history\"),\n",
|
||||
" (\"user\", \"{input}\"),\n",
|
||||
" MessagesPlaceholder(variable_name=\"agent_scratchpad\"),\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"llm_with_tools = llm.bind(functions=[format_tool_to_openai_function(t) for t in tools])\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _format_chat_history(chat_history: List[Tuple[str, str]]):\n",
|
||||
" buffer = []\n",
|
||||
" for human, ai in chat_history:\n",
|
||||
" buffer.append(HumanMessage(content=human))\n",
|
||||
" buffer.append(AIMessage(content=ai))\n",
|
||||
" return buffer\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"agent = (\n",
|
||||
" {\n",
|
||||
" \"input\": lambda x: x[\"input\"],\n",
|
||||
" \"chat_history\": lambda x: _format_chat_history(x[\"chat_history\"]),\n",
|
||||
" \"agent_scratchpad\": lambda x: format_to_openai_functions(\n",
|
||||
" x[\"intermediate_steps\"]\n",
|
||||
" ),\n",
|
||||
" }\n",
|
||||
" | prompt\n",
|
||||
" | llm_with_tools\n",
|
||||
" | OpenAIFunctionsAgentOutputParser()\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class AgentInput(BaseModel):\n",
|
||||
" input: str\n",
|
||||
" chat_history: List[Tuple[str, str]] = Field(..., extra={\"widget\": {\"type\": \"chat\"}})\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=False).with_types(\n",
|
||||
" input_type=AgentInput\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class ChainInput(BaseModel):\n",
|
||||
" question: str\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def mapper(input: dict):\n",
|
||||
" return {\"input\": input[\"question\"], \"chat_history\": []}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"agent_executor = (mapper | agent_executor | (lambda x: x[\"output\"])).with_types(\n",
|
||||
" input_type=ChainInput\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5a1c09c2-a983-450b-a531-c6871a9b27ae",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"oai_functions_test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=agent_executor,\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"oai-functions basic rag-simple {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"basic\",\n",
|
||||
" \"embedding_model\": \"thenlper/gte-base\",\n",
|
||||
" \"llm\": \"gpt-4-1106-preview\",\n",
|
||||
" \"architecture\": \"oai-functions-agent\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "73ffd740-fde1-479d-b84c-7dd8f65716a6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Assistant\n",
|
||||
"\n",
|
||||
"OpenAI provides a hosted agent service through their Assistants API. \n",
|
||||
"\n",
|
||||
"You can connect your LangChain retriever to an OpenAI's Assistant API and evaluate its performance. Let's test below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3de4d42b-e34a-4980-97eb-9b2c78a24089",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"from langchain.agents import AgentExecutor\n",
|
||||
"from langchain.tools import tool\n",
|
||||
"from langchain_experimental.openai_assistant import OpenAIAssistantRunnable\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"@tool\n",
|
||||
"def search(query, callbacks=None) -> str:\n",
|
||||
" \"\"\"Search the LangChain docs with the retriever.\"\"\"\n",
|
||||
" docs = retriever.get_relevant_documents(query, callbacks=callbacks)\n",
|
||||
" return json.dumps([doc.dict() for doc in docs])\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tools = [search]\n",
|
||||
"\n",
|
||||
"agent = OpenAIAssistantRunnable.create_assistant(\n",
|
||||
" name=\"langchain docs assistant\",\n",
|
||||
" instructions=\"You are a helpful assistant tasked with answering technical questions about LangChain.\",\n",
|
||||
" tools=tools,\n",
|
||||
" model=\"gpt-4-1106-preview\",\n",
|
||||
" as_agent=True,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"assistant_exector = (\n",
|
||||
" (lambda x: {\"content\": x[\"question\"]})\n",
|
||||
" | AgentExecutor(agent=agent, tools=tools)\n",
|
||||
" | (lambda x: x[\"output\"])\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cd6baea1-ac90-43aa-a21c-98aa5ca23732",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"assistant_test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=assistant_exector,\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"oai-assistant basic rag-simple {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"basic\",\n",
|
||||
" \"embedding_model\": \"thenlper/gte-base\",\n",
|
||||
" \"llm\": \"gpt-4-1106-preview\",\n",
|
||||
" \"architecture\": \"oai-assistant\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a5ac5fad-0a74-4403-a917-7145be6d7d1a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"assistant_test_run.get_aggregate_feedback()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6a04b3e4-b5df-4075-9089-8aa10ef63348",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Introduction\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"These tasks are meant to test retrieval-augmented generation (RAG) architectures on various datasets.\n",
|
||||
"\n",
|
||||
"You can check an up-to-date list of retrieval tasks in the registry:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<thead>\n",
|
||||
"<tr><th>Name </th><th>Type </th><th>Dataset ID </th><th>Description </th></tr>\n",
|
||||
"</thead>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>LangChain Docs Q&A </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
|
||||
"\n",
|
||||
"The environment provides the documents and the retriever information.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"<tr><td>Semi-structured Earnings</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
|
||||
"\n",
|
||||
"The task provides the raw documents as well as factory methods to easily index them\n",
|
||||
"and create a retriever.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x138367ec0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x138367f60>, 'hyde': <function _chroma_hyde_retriever_factory at 0x13838c040>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x11fa74fe0>}, get_docs=<function load_cached_docs at 0x101bfb240>), RetrievalTask(name='Semi-structured Earnings', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x13838c5e0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x13838c680>, 'hyde': <function _chroma_hyde_retriever_factory at 0x13838c720>}, architecture_factories={}, get_docs=<function load_docs at 0x13838c540>)])"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_benchmarks import registry\n",
|
||||
"\n",
|
||||
"registry.filter(Type=\"RetrievalTask\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Task resources\n",
|
||||
"\n",
|
||||
"In addition to a name, daset_id, and description, each retrieval task provides a few helper functions you can use to configure your pipeline.\n",
|
||||
"\n",
|
||||
"- `get_docs: callable` - fetches the original `Document` objects from the cache. Each task may provide configurable parameters you can use to define how the original documents are fetched.\n",
|
||||
"- `retriever_factories: Dict[str, callable]` - define some configurable pipelines you can use to transform the documents, embed them, and add them to a vectorstore (or other retriever object) for downstream use. They use LangChain's caching `index` API so you don't have to re-index for every evaluation. For custom transformations, we ask that you provide a `transformation_name` to isolate the cache and vectorstore namespace. Currently (2023/11/21) these all use Chroma as a vectorstore, but you can swap this out\n",
|
||||
"- `chain_factories: Dict[str, callable]` - define some off-the-shelf architectures you can configure to evaluate.\n",
|
||||
"\n",
|
||||
"When evaluating, you don't have to use any of these factory methods. You can instead define your own custom architecture or ETl pipeline before evaluating. They are meant to facilitate evaluations and comparisons for specific design decisions.\n",
|
||||
"\n",
|
||||
"### Dataset schema\n",
|
||||
"\n",
|
||||
"Each task corresponds to a LangSmith dataset with the following schema:\n",
|
||||
"\n",
|
||||
"Inputs:\n",
|
||||
"- question: str - the user question\n",
|
||||
"\n",
|
||||
"Outputs\n",
|
||||
"- answer: str - the expected answer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
@@ -1,518 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Q&A over LangChain Docs\n",
|
||||
"\n",
|
||||
"Let's evaluate your architecture on a Q&A dataset for the LangChain python docs. For more examples of how to test different embeddings, indexing strategies, and architectures, see the [Evaluating RAG Architectures on Benchmark Tasks](./comparing_techniques.ipynb) notebook.\n",
|
||||
"\n",
|
||||
"## Pre-requisites\n",
|
||||
"\n",
|
||||
"We will install quite a few prerequisites for this example since we are comparing many techniques and models.\n",
|
||||
"\n",
|
||||
"We will be using LangSmith to capture the evaluation traces. You can make a free account at [smith.langchain.com](https://smith.langchain.com/). Once you've done so, you can make an API key and set it below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9f44b59b",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install -U --quiet langchain langsmith langchainhub langchain_benchmarks\n",
|
||||
"%pip install --quiet chromadb openai huggingface pandas langchain_experimental sentence_transformers pyarrow anthropic tiktoken"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0aae13f6-cd40-41e6-bd02-bd683e91cbff",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"For this code to work, please configure LangSmith environment variables with your credentials."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "62b518cf-99fb-44be-8acb-ee0a8ba62272",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
|
||||
"os.environ[\"LANGCHAIN_API_KEY\"] = \"ls_...\" # Your API key"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "4292ee8c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Update these with your own API keys\n",
|
||||
"os.environ[\"ANTHROPIC_API_KEY\"] = \"sk-...\"\n",
|
||||
"os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
|
||||
"# Silence warnings from HuggingFace\n",
|
||||
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "1ccbf17a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"# Generate a unique run ID for this experiment\n",
|
||||
"run_uid = uuid.uuid4().hex[:6]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2e8a666d-8bf5-4bfd-8b20-8b7defdb8cd5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Review Q&A Tasks\n",
|
||||
"\n",
|
||||
"The registry provides configurations to test out common architectures on curated datasets. You can review retrieval tasks by filtering on the Type."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks import clone_public_dataset, registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "3644d211-382e-41aa-b282-21b01d28fc35",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<thead>\n",
|
||||
"<tr><th>Name </th><th>Type </th><th>Dataset ID </th><th>Description </th></tr>\n",
|
||||
"</thead>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>LangChain Docs Q&A </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
|
||||
"\n",
|
||||
"The environment provides the documents and the retriever information.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"<tr><td>Semi-structured Reports</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
|
||||
"\n",
|
||||
"The task provides the raw documents as well as factory methods to easily index them\n",
|
||||
"and create a retriever.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x2b433fc40>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x2b433fce0>, 'hyde': <function _chroma_hyde_retriever_factory at 0x2b433fd80>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x2b3b59580>}, get_docs=<function load_cached_docs at 0x2b3b59080>), RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x2b43644a0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x2b4364540>, 'hyde': <function _chroma_hyde_retriever_factory at 0x2b43645e0>}, architecture_factories={}, get_docs=<function load_docs at 0x2b4364400>)])"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"registry = registry.filter(Type=\"RetrievalTask\")\n",
|
||||
"registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "671282f8-c455-4390-b018-e53bbd833093",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>LangChain Docs Q&A </td></tr>\n",
|
||||
"<tr><td>Type </td><td>RetrievalTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td></tr>\n",
|
||||
"<tr><td>Description </td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
|
||||
"\n",
|
||||
"The environment provides the documents and the retriever information.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"<tr><td>Retriever Factories </td><td>basic, parent-doc, hyde </td></tr>\n",
|
||||
"<tr><td>Architecture Factories</td><td>conversational-retrieval-qa </td></tr>\n",
|
||||
"<tr><td>get_docs </td><td><function load_cached_docs at 0x2b3b59080> </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x2b433fc40>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x2b433fce0>, 'hyde': <function _chroma_hyde_retriever_factory at 0x2b433fd80>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x2b3b59580>}, get_docs=<function load_cached_docs at 0x2b3b59080>)"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"langchain_docs = registry[\"LangChain Docs Q&A\"]\n",
|
||||
"langchain_docs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "968f78bb-7e00-407f-b486-1e5b27f3287a",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Clone the dataset\n",
|
||||
"\n",
|
||||
"Once you've selected the LangChain Docs Q&A task, clone the dataset to your LangSmith tenant. This step\n",
|
||||
"requires that your LANGCHAIN_API_KEY be set above."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "70369f67-deb4-467a-801a-6d38c3d0460d",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dataset LangChain Docs Q&A already exists. Skipping.\n",
|
||||
"You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/3f29798f-5939-4643-bd99-008ca66b72ed.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clone_public_dataset(langchain_docs.dataset_id, dataset_name=langchain_docs.name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2867f083-1f36-40a9-8e97-74381d47afcd",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Create the index\n",
|
||||
"\n",
|
||||
"When creating a retrieval Q&A system, the first step is to prepare the retriever. How you construct the index significantly impact your system's performance. Before trying anything too tricky, it's good benchmark a reliable baseline.\n",
|
||||
"\n",
|
||||
"In this case, our baseline will be to generate a single vector for each raw source document and store them directly in a vector store.\n",
|
||||
"\n",
|
||||
"Below, fetch the source docs from the cache in GCS. This cache was formed using an [ingestion script](https://github.com/langchain-ai/langchain-benchmarks/blob/30aa706d9cefbaebb219f1763c04fafab6d0ee78/langchain_benchmarks/rag/tasks/langchain_docs/_ingest_docs.py#L1) that scraped the LangChain documentation. To save time and to ensure that the dataset answers are still correct, we will use these source docs for all benchmark approaches."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"id": "3afe1e54-9354-4bbc-9fd8-0f9d769fc43d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Document(page_content=\"LangChain cookbook | 🦜️🔗 Langchain\\n\\n[Skip to main content](#docusaurus_skip...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"docs = list(langchain_docs.get_docs())\n",
|
||||
"print(repr(docs[0])[:100] + \"...\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f7e6830a-dd2f-4532-95e7-81353d18dcd2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now we will populate our vectorstore. We will use LangChain's indexing API to cache embeddings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"id": "4980f8d4-d4e6-4b70-a04d-a7d9333545ee",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.embeddings import HuggingFaceEmbeddings\n",
|
||||
"from langchain.vectorstores.chroma import Chroma\n",
|
||||
"\n",
|
||||
"embeddings = HuggingFaceEmbeddings(\n",
|
||||
" model_name=\"thenlper/gte-base\",\n",
|
||||
" # model_kwargs={\"device\": 0}, # Comment out to use CPU\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"vectorstore = Chroma(\n",
|
||||
" collection_name=\"lcbm-b-huggingface-gte-base\",\n",
|
||||
" embedding_function=embeddings,\n",
|
||||
" persist_directory=\"./chromadb\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"vectorstore.add_documents(docs)\n",
|
||||
"retriever = vectorstore.as_retriever(search_kwargs={\"k\": 6})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "54bac7ea-ab28-4b08-9d06-56ac869085dd",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Define the response generator\n",
|
||||
"\n",
|
||||
"Halfway done with our RAG system. We've made the **R**etriever. Now time for the response **G**enerator.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"id": "41e64350-63a7-4e7d-8e03-7dc459c444cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from operator import itemgetter\n",
|
||||
"from typing import Sequence\n",
|
||||
"\n",
|
||||
"from langchain.chat_models import ChatAnthropic\n",
|
||||
"from langchain.prompts import ChatPromptTemplate\n",
|
||||
"from langchain.schema.document import Document\n",
|
||||
"from langchain.schema.output_parser import StrOutputParser\n",
|
||||
"from langchain.schema.runnable.passthrough import RunnableAssign\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# After the retriever fetches documents, this\n",
|
||||
"# function formats them in a string to present for the LLM\n",
|
||||
"def format_docs(docs: Sequence[Document]) -> str:\n",
|
||||
" formatted_docs = []\n",
|
||||
" for i, doc in enumerate(docs):\n",
|
||||
" doc_string = (\n",
|
||||
" f\"<document index='{i}'>\\n\"\n",
|
||||
" f\"<source>{doc.metadata.get('source')}</source>\\n\"\n",
|
||||
" f\"<doc_content>{doc.page_content}</doc_content>\\n\"\n",
|
||||
" \"</document>\"\n",
|
||||
" )\n",
|
||||
" formatted_docs.append(doc_string)\n",
|
||||
" formatted_str = \"\\n\".join(formatted_docs)\n",
|
||||
" return f\"<documents>\\n{formatted_str}\\n</documents>\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\n",
|
||||
" \"system\",\n",
|
||||
" \"You are an AI assistant answering questions about LangChain.\"\n",
|
||||
" \"\\n{context}\\n\"\n",
|
||||
" \"Respond solely based on the document content.\",\n",
|
||||
" ),\n",
|
||||
" (\"human\", \"{question}\"),\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"llm = ChatAnthropic(model=\"claude-2.1\", temperature=1)\n",
|
||||
"\n",
|
||||
"response_generator = (prompt | llm | StrOutputParser()).with_config(\n",
|
||||
" run_name=\"GenerateResponse\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# This is the final response chain.\n",
|
||||
"# It fetches the \"question\" key from the input dict,\n",
|
||||
"# passes it to the retriever, then formats as a string.\n",
|
||||
"\n",
|
||||
"chain = (\n",
|
||||
" RunnableAssign(\n",
|
||||
" {\n",
|
||||
" \"context\": (itemgetter(\"question\") | retriever | format_docs).with_config(\n",
|
||||
" run_name=\"FormatDocs\"\n",
|
||||
" )\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
" # The \"RunnableAssign\" above returns a dict with keys\n",
|
||||
" # question (from the original input) and\n",
|
||||
" # context: the string-formatted docs.\n",
|
||||
" # This is passed to the response_generator above\n",
|
||||
" | response_generator\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"id": "10a1fca9-d356-4cff-93a9-c4f63944e57d",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"' The LangChain Expression Language (LCEL) is a declarative way to easily compose chains of different components like prompts, models, parsers, etc. \\n\\nSome key things it provides:\\n\\n- Streaming support - Ability to get incremental outputs from chains rather than waiting for full completion. Useful for long-running chains.\\n\\n- Async support - Chains can be called synchronously (like in a notebook) or asynchronously (like in production). Same code works for both.\\n\\n- Optimized parallel execution - Steps that can run in parallel (like multiple retrievals) are automatically parallelized to minimize latency.\\n\\n- Retries and fallbacks - Help make chains more robust to failure.\\n\\n- Access to intermediate results - Useful for debugging or showing work-in-progress.\\n\\n- Input and output validation via schemas - Enables catching issues early.\\n\\n- Tracing - Automatic structured logging of all chain steps for observability.\\n\\n- Seamless deployment - LCEL chains can be easily deployed with LangServe.\\n\\nThe key idea is it makes it very easy to take a prototype LLM application made with components like prompts and models and turn it into a robust, scalable production application without changing any code.'"
|
||||
]
|
||||
},
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"chain.invoke({\"question\": \"What's expression language?\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3821e4b0-8e67-418a-840c-470fcde42df0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Evaluate\n",
|
||||
"\n",
|
||||
"Let's evaluate your RAG architecture on the dataset now."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"id": "513042fe-2878-44f8-ae84-05b9d521c1de",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.rag import get_eval_config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"id": "aab7514e-a6ef-4c21-b90f-d9cbefcf5af1",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'claude-2 qa-chain simple-index 1bdbe5' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/3fe31959-95e8-4413-aa09-620bd49bd0d3?eval=true\n",
|
||||
"\n",
|
||||
"View all tests for Dataset LangChain Docs Q&A at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/3f29798f-5939-4643-bd99-008ca66b72ed\n",
|
||||
"[------------------------------------------------->] 86/86\n",
|
||||
" Eval quantiles:\n",
|
||||
" 0.25 0.5 0.75 mean \\\n",
|
||||
"embedding_cosine_distance 0.088025 0.115760 0.159969 0.129161 \n",
|
||||
"score_string:accuracy 0.500000 0.700000 1.000000 0.645349 \n",
|
||||
"faithfulness 0.700000 1.000000 1.000000 0.812791 \n",
|
||||
"execution_time 27.098772 27.098772 27.098772 27.098772 \n",
|
||||
"\n",
|
||||
" mode \n",
|
||||
"embedding_cosine_distance 0.048622 \n",
|
||||
"score_string:accuracy 0.700000 \n",
|
||||
"faithfulness 1.000000 \n",
|
||||
"execution_time 27.098772 \n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"client = Client()\n",
|
||||
"RAG_EVALUATION = get_eval_config()\n",
|
||||
"\n",
|
||||
"test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=langchain_docs.name,\n",
|
||||
" llm_or_chain_factory=chain,\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" project_name=f\"claude-2 qa-chain simple-index {run_uid}\",\n",
|
||||
" project_metadata={\n",
|
||||
" \"index_method\": \"basic\",\n",
|
||||
" },\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "79d85a82-5ba7-479f-94e2-c9e3c7cb90cb",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_run.get_aggregate_feedback()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "230bf290-0422-4774-b82f-f78eb2e51918",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
-610
@@ -1,610 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9fa3470d-9448-4792-9f65-6978fc58cf84",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Multi-modal eval: Baseline\n",
|
||||
"\n",
|
||||
"`Multi-modal slide decks` is a public dataset that contains a dataset of question-answer pairs from slide decks with visual content.\n",
|
||||
"\n",
|
||||
"The question-answer pairs are derived from the visual content in the decks, testing the ability of RAG to perform visual reasoning.\n",
|
||||
"\n",
|
||||
"As a baseline, we evaluate this dataset using text-based RAG pipeline, below.\n",
|
||||
"\n",
|
||||
"This will not reason about visual content and will simply load the text from the slides. \n",
|
||||
"\n",
|
||||
"## Pre-requisites"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "47220461-d4e9-4f1d-9c57-672ca947ca0d",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# %pip install -U langchain langsmith langchain_benchmarks\n",
|
||||
"# %pip install --quiet chromadb openai pypdf pandas"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "196de967-6de6-40da-aa75-e836923ab5e3",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import getpass\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
|
||||
"env_vars = [\"LANGCHAIN_API_KEY\", \"OPENAI_API_KEY\"]\n",
|
||||
"for var in env_vars:\n",
|
||||
" if var not in os.environ:\n",
|
||||
" os.environ[var] = getpass.getpass(prompt=f\"Enter your {var}: \")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "10da8e11-6288-4131-bd60-d5aa86928acc",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Dataset\n",
|
||||
"\n",
|
||||
"We can browse the available LangChain benchmark datasets for retrieval."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "2ff97905-14a6-413c-99be-58b7a9c8d4c1",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<thead>\n",
|
||||
"<tr><th>Name </th><th>Type </th><th>Dataset ID </th><th>Description </th></tr>\n",
|
||||
"</thead>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>LangChain Docs Q&A </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
|
||||
"\n",
|
||||
"The environment provides the documents and the retriever information.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"<tr><td>Semi-structured Reports</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
|
||||
"\n",
|
||||
"The task provides the raw documents as well as factory methods to easily index them\n",
|
||||
"and create a retriever.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"<tr><td>Multi-modal slide decks</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/40afc8e7-9d7e-44ed-8971-2cae1eb59731/d\" target=\"_blank\" rel=\"noopener\">40afc8e7-9d7e-44ed-8971-2cae1eb59731</a></td><td>This public dataset is a work-in-progress and will be extended over time.\n",
|
||||
" \n",
|
||||
"Questions and answers based on slide decks containing visual tables and charts.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", get_docs=<function load_cached_docs at 0x104485800>, retriever_factories={'basic': <function _chroma_retriever_factory at 0x1360289a0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x136028a40>, 'hyde': <function _chroma_hyde_retriever_factory at 0x136028ae0>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x126ba2660>}), RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", get_docs=<function load_docs at 0x136029620>, retriever_factories={'basic': <function _chroma_retriever_factory at 0x1360296c0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x136029760>, 'hyde': <function _chroma_hyde_retriever_factory at 0x136029800>}, architecture_factories={}), RetrievalTask(name='Multi-modal slide decks', dataset_id='https://smith.langchain.com/public/40afc8e7-9d7e-44ed-8971-2cae1eb59731/d', description='This public dataset is a work-in-progress and will be extended over time.\\n \\nQuestions and answers based on slide decks containing visual tables and charts.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\n', get_docs={}, retriever_factories={}, architecture_factories={})])"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_benchmarks import clone_public_dataset, registry\n",
|
||||
"\n",
|
||||
"registry = registry.filter(Type=\"RetrievalTask\")\n",
|
||||
"registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2fb7dc3d-28f1-4c28-b0d0-3784d04b81ce",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"`Multi-modal slide decks` is the relevant dataset for our task."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "219a4141-4a5f-48e4-ae05-5a824e2193fd",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>Multi-modal slide decks </td></tr>\n",
|
||||
"<tr><td>Type </td><td>RetrievalTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/40afc8e7-9d7e-44ed-8971-2cae1eb59731/d\" target=\"_blank\" rel=\"noopener\">40afc8e7-9d7e-44ed-8971-2cae1eb59731</a></td></tr>\n",
|
||||
"<tr><td>Description </td><td>This public dataset is a work-in-progress and will be extended over time.\n",
|
||||
" \n",
|
||||
"Questions and answers based on slide decks containing visual tables and charts.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer. </td></tr>\n",
|
||||
"<tr><td>Retriever Factories </td><td> </td></tr>\n",
|
||||
"<tr><td>Architecture Factories</td><td> </td></tr>\n",
|
||||
"<tr><td>get_docs </td><td>{} </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"RetrievalTask(name='Multi-modal slide decks', dataset_id='https://smith.langchain.com/public/40afc8e7-9d7e-44ed-8971-2cae1eb59731/d', description='This public dataset is a work-in-progress and will be extended over time.\\n \\nQuestions and answers based on slide decks containing visual tables and charts.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\n', get_docs={}, retriever_factories={}, architecture_factories={})"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task = registry[\"Multi-modal slide decks\"]\n",
|
||||
"task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2d6569b5-e79a-41b7-9745-c2f8a1dd704e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Clone the dataset so that it's available in our LangSmith datasets."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "d2caa086-9549-4c74-bba9-ba80d5a7b218",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dataset Multi-modal slide decks already exists. Skipping.\n",
|
||||
"You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/08a29acb-5ad6-42ce-a482-574c9e2e5306.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clone_public_dataset(task.dataset_id, dataset_name=task.name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bf350917-a1e5-46f4-81cd-c1678ab9220f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Fetch the associated PDFs from remote cache for the dataset so that we can perform ingestion."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "99ce6afb-2317-4bc1-9faf-4f828095ad91",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks.rag.tasks.multi_modal_slide_decks import get_file_names\n",
|
||||
"\n",
|
||||
"file_names = list(get_file_names()) # PosixPath"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "848a4cdb-6c08-4c01-81ce-16ab83a7fdff",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load\n",
|
||||
"\n",
|
||||
"Load and split the files for indexing."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "6ce85810-98a7-406e-b44e-ce860ac35986",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"There are 98 text elements in DDOG_Q3_earnings_deck.pdf\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain.document_loaders import PyPDFLoader\n",
|
||||
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_and_split(file):\n",
|
||||
" \"\"\"\n",
|
||||
" Load and split PDF files\n",
|
||||
" :param file: PosixPath path for pdf\n",
|
||||
" :return: A list of text chunks\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" loader = PyPDFLoader(str(file))\n",
|
||||
" pdf_pages = loader.load()\n",
|
||||
"\n",
|
||||
" text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
|
||||
" chunk_size=100, chunk_overlap=50\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Get chunks\n",
|
||||
" docs = text_splitter.split_documents(pdf_pages)\n",
|
||||
" texts = [d.page_content for d in docs]\n",
|
||||
" print(f\"There are {len(texts)} text elements in {file.name}\")\n",
|
||||
" return texts\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"texts = []\n",
|
||||
"for fi in file_names:\n",
|
||||
" texts.extend(load_and_split(fi))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "eb01925d-b7d1-47a1-bd90-805178d3c4a9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Index\n",
|
||||
"\n",
|
||||
"Embed (OpenAIEmbeddings) and store splits in a vectorstore (Chroma)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "ceb31f71-45fb-4b12-bc1c-31981de334bb",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.embeddings import OpenAIEmbeddings\n",
|
||||
"from langchain.vectorstores import Chroma\n",
|
||||
"\n",
|
||||
"vectorstore_baseline = Chroma.from_texts(\n",
|
||||
" texts=texts, collection_name=\"baseline-multi-modal\", embedding=OpenAIEmbeddings()\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"retriever_baseline = vectorstore_baseline.as_retriever()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e6dcbb01-f480-456d-b972-c732eb26c393",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## RAG\n",
|
||||
"\n",
|
||||
"Create a pipeline for retrieval of relevant chunks based on semantic similarity to the input question.\n",
|
||||
"\n",
|
||||
"Pass the images to GPT-4 for answer synthesis."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "ea233664-e527-42f1-a820-0c2271e16c20",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.prompts import ChatPromptTemplate\n",
|
||||
"from langchain.schema.output_parser import StrOutputParser\n",
|
||||
"from langchain.schema.runnable import RunnablePassthrough\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def rag_chain(retriever):\n",
|
||||
" \"\"\"\n",
|
||||
" RAG pipeline for the indexed presentations\n",
|
||||
" :param retriever: PosixPath path for pdf\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" # Prompt template\n",
|
||||
" template = \"\"\"Answer the question based only on the following context, which can include text and tables:\n",
|
||||
" {context}\n",
|
||||
" Question: {question}\n",
|
||||
" \"\"\"\n",
|
||||
" prompt = ChatPromptTemplate.from_template(template)\n",
|
||||
"\n",
|
||||
" # LLM\n",
|
||||
" model = ChatOpenAI(temperature=0, model=\"gpt-4\")\n",
|
||||
"\n",
|
||||
" # RAG pipeline\n",
|
||||
" chain = (\n",
|
||||
" {\n",
|
||||
" \"context\": retriever | (lambda x: \"\\n\\n\".join([i.page_content for i in x])),\n",
|
||||
" \"question\": RunnablePassthrough(),\n",
|
||||
" }\n",
|
||||
" | prompt\n",
|
||||
" | model\n",
|
||||
" | StrOutputParser()\n",
|
||||
" )\n",
|
||||
" return chain\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Create RAG chain\n",
|
||||
"chain = rag_chain(retriever_baseline)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "95df1446-143d-4f4c-a15b-2a379266d8bf",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Eval\n",
|
||||
"\n",
|
||||
"Run evaluation on our dataset:\n",
|
||||
"\n",
|
||||
"* `task.name` is the dataset of QA pairs that we cloned\n",
|
||||
"* `eval_config` specifies the [LangSmith evaluator](https://docs.smith.langchain.com/evaluation/evaluator-implementations#correctness-qa-evaluation) for our dataset, which will use GPT-4 as a grader\n",
|
||||
"* The grader will evaluate the chain-generated answer to each question relative to ground truth"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "479ce09d-642e-4b3b-9e4e-e9c2b7f0e9ca",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project '866f-baseline' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/08a29acb-5ad6-42ce-a482-574c9e2e5306/compare?selectedSessions=30199d47-50d7-4c5c-a55a-e74157e05951\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Multi-modal slide decks at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/08a29acb-5ad6-42ce-a482-574c9e2e5306\n",
|
||||
"[------------------------------------------------->] 10/10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<h3>Experiment Results:</h3>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>output</th>\n",
|
||||
" <th>feedback.COT Contextual Accuracy</th>\n",
|
||||
" <th>error</th>\n",
|
||||
" <th>execution_time</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>count</th>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>10.000000</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>10.000000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>unique</th>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>top</th>\n",
|
||||
" <td>Datadog has 20 total customers.</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>freq</th>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>mean</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.200000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>4.674478</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>std</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.421637</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.864273</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>min</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.000000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>3.307960</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>25%</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.000000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>4.113816</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>50%</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.000000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>4.700962</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>75%</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.000000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>5.018359</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>max</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>1.000000</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>6.188082</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" output feedback.COT Contextual Accuracy \\\n",
|
||||
"count 10 10.000000 \n",
|
||||
"unique 10 NaN \n",
|
||||
"top Datadog has 20 total customers. NaN \n",
|
||||
"freq 1 NaN \n",
|
||||
"mean NaN 0.200000 \n",
|
||||
"std NaN 0.421637 \n",
|
||||
"min NaN 0.000000 \n",
|
||||
"25% NaN 0.000000 \n",
|
||||
"50% NaN 0.000000 \n",
|
||||
"75% NaN 0.000000 \n",
|
||||
"max NaN 1.000000 \n",
|
||||
"\n",
|
||||
" error execution_time \n",
|
||||
"count 0 10.000000 \n",
|
||||
"unique 0 NaN \n",
|
||||
"top NaN NaN \n",
|
||||
"freq NaN NaN \n",
|
||||
"mean NaN 4.674478 \n",
|
||||
"std NaN 0.864273 \n",
|
||||
"min NaN 3.307960 \n",
|
||||
"25% NaN 4.113816 \n",
|
||||
"50% NaN 4.700962 \n",
|
||||
"75% NaN 5.018359 \n",
|
||||
"max NaN 6.188082 "
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"from langchain.smith import RunEvalConfig\n",
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"# Evaluator configuration\n",
|
||||
"client = Client()\n",
|
||||
"eval_config = RunEvalConfig(\n",
|
||||
" evaluators=[\"cot_qa\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Experiments\n",
|
||||
"chain_map = {\n",
|
||||
" \"baseline\": chain,\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Run evaluation\n",
|
||||
"run_id = uuid.uuid4().hex[:4]\n",
|
||||
"test_runs = {}\n",
|
||||
"for project_name, chain in chain_map.items():\n",
|
||||
" test_runs[project_name] = client.run_on_dataset(\n",
|
||||
" dataset_name=task.name,\n",
|
||||
" llm_or_chain_factory=lambda: (lambda x: x[\"Question\"]) | chain,\n",
|
||||
" evaluation=eval_config,\n",
|
||||
" verbose=True,\n",
|
||||
" project_name=f\"{run_id}-{project_name}\",\n",
|
||||
" project_metadata={\"chain\": project_name},\n",
|
||||
" )"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,645 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Semi-structured RAG\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Let's evaluate your architecture on a small semi-structured Q&A dataset. This dataset is composed of QA pairs over pdfs that contain tables."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f49db759-7ce6-4ab7-a58f-7fc3a6a7c8ec",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Pre-requisites\n",
|
||||
"\n",
|
||||
"We will install quite a few prerequisites for this example since we are comparing various techinques and models."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "9f44b59b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install -U langchain langsmith langchainhub langchain_benchmarks langchain_experimental\n",
|
||||
"%pip install --quiet chromadb openai huggingface pandas \"unstructured[all-docs]\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0aae13f6-cd40-41e6-bd02-bd683e91cbff",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"For this code to work, please configure LangSmith environment variables with your credentials."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "62b518cf-99fb-44be-8acb-ee0a8ba62272",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
|
||||
"os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\" # Your API key\n",
|
||||
"\n",
|
||||
"# Silence warnings from HuggingFace\n",
|
||||
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2e8a666d-8bf5-4bfd-8b20-8b7defdb8cd5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Review Q&A Tasks\n",
|
||||
"\n",
|
||||
"The registry provides configurations to test out common architectures on curated datasets."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks import clone_public_dataset, registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "3644d211-382e-41aa-b282-21b01d28fc35",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<thead>\n",
|
||||
"<tr><th>Name </th><th>Type </th><th>Dataset ID </th><th>Description </th></tr>\n",
|
||||
"</thead>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>LangChain Docs Q&A </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
|
||||
"\n",
|
||||
"The environment provides the documents and the retriever information.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"<tr><td>Semi-structured Reports</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
|
||||
"\n",
|
||||
"The task provides the raw documents as well as factory methods to easily index them\n",
|
||||
"and create a retriever.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x122c7c4a0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x122c7c540>, 'hyde': <function _chroma_hyde_retriever_factory at 0x122c7c5e0>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x12233be20>}, get_docs=<function load_cached_docs at 0x12233b920>), RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x122c7ccc0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x122c7cd60>, 'hyde': <function _chroma_hyde_retriever_factory at 0x122c7ce00>}, architecture_factories={}, get_docs=<function load_docs at 0x122c7cc20>)])"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"registry = registry.filter(Type=\"RetrievalTask\")\n",
|
||||
"registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "671282f8-c455-4390-b018-e53bbd833093",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>Semi-structured Reports </td></tr>\n",
|
||||
"<tr><td>Type </td><td>RetrievalTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td></tr>\n",
|
||||
"<tr><td>Description </td><td>Questions and answers based on PDFs containing tables and charts.\n",
|
||||
"\n",
|
||||
"The task provides the raw documents as well as factory methods to easily index them\n",
|
||||
"and create a retriever.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question and reference answer.\n",
|
||||
"\n",
|
||||
"Success is measured based on the accuracy of the answer relative to the reference answer.\n",
|
||||
"We also measure the faithfulness of the model's response relative to the retrieved documents (if any). </td></tr>\n",
|
||||
"<tr><td>Retriever Factories </td><td>basic, parent-doc, hyde </td></tr>\n",
|
||||
"<tr><td>Architecture Factories</td><td> </td></tr>\n",
|
||||
"<tr><td>get_docs </td><td><function load_docs at 0x122c7cc20> </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x122c7ccc0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x122c7cd60>, 'hyde': <function _chroma_hyde_retriever_factory at 0x122c7ce00>}, architecture_factories={}, get_docs=<function load_docs at 0x122c7cc20>)"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task = registry[\"Semi-structured Reports\"]\n",
|
||||
"task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "70369f67-deb4-467a-801a-6d38c3d0460d",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dataset Semi-structured Reports already exists. Skipping.\n",
|
||||
"You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/f8f24935-cf57-4cb3-a30f-8df303a46962.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clone_public_dataset(task.dataset_id, dataset_name=task.name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4b4fafb2-63d0-40b4-b803-0095c5b22ca6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Now, index the documents\n",
|
||||
"\n",
|
||||
"You can see the raw filepaths, or use unstructured to process the pdfs."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "c9657b27-4e10-4ba5-ab20-1f05f22fdbd4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks.rag.tasks.semi_structured_reports import get_file_names\n",
|
||||
"\n",
|
||||
"# If you want to completely customize the document processing, you can use the files directly\n",
|
||||
"file_names = list(get_file_names())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "f5ad4b23-fbd4-4ebc-b5a5-d3d05efd0b9c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of the model checkpoint at microsoft/table-transformer-structure-recognition were not used when initializing TableTransformerForObjectDetection: ['model.backbone.conv_encoder.model.layer2.0.downsample.1.num_batches_tracked', 'model.backbone.conv_encoder.model.layer3.0.downsample.1.num_batches_tracked', 'model.backbone.conv_encoder.model.layer4.0.downsample.1.num_batches_tracked']\n",
|
||||
"- This IS expected if you are initializing TableTransformerForObjectDetection from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
||||
"- This IS NOT expected if you are initializing TableTransformerForObjectDetection from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b8a52c9983274c21a713ac8742e9c99b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/26 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain.embeddings import HuggingFaceEmbeddings\n",
|
||||
"\n",
|
||||
"embeddings = HuggingFaceEmbeddings(\n",
|
||||
" model_name=\"thenlper/gte-base\",\n",
|
||||
" model_kwargs={\"device\": 0}, # Comment out to use CPU\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Arguments to pass to partition_pdf\n",
|
||||
"unstructured_config = {\n",
|
||||
" # Unstructured first finds embedded image blocks\n",
|
||||
" \"extract_images_in_pdf\": False,\n",
|
||||
" # Use layout model (YOLOX) to get bounding boxes (for tables) and find titles\n",
|
||||
" # Titles are any sub-section of the document\n",
|
||||
" \"infer_table_structure\": True,\n",
|
||||
" # Post processing to aggregate text once we have the title\n",
|
||||
" \"chunking_strategy\": \"by_title\",\n",
|
||||
" # Chunking params to aggregate text blocks\n",
|
||||
" # Attempt to create a new chunk 3800 chars\n",
|
||||
" # Attempt to keep chunks > 2000 chars\n",
|
||||
" \"max_characters\": 4000,\n",
|
||||
" \"new_after_n_chars\": 3800,\n",
|
||||
" \"combine_text_under_n_chars\": 2000,\n",
|
||||
"}\n",
|
||||
"docs = list(task.get_docs(unstructured_config=unstructured_config))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 42,
|
||||
"id": "fe0a05ad-5b57-40b0-aac4-e2d9cd9e6b4b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Chroma/semi-structured-earnings-b_Chroma_HuggingFaceEmbeddings_raw\n",
|
||||
"[]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "06a11e1a4d50416596d9dd953fdabafa",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/26 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"retriever_factory = task.retriever_factories[\"basic\"]\n",
|
||||
"# Indexes the documents with the specified embeddings\n",
|
||||
"retriever = retriever_factory(embeddings, docs=docs)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "57efac89-12f9-47e3-b60f-65d9279ebc1e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Time to evaluate\n",
|
||||
"\n",
|
||||
"We will compose our retriever with a simple Llama based LLM."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "8d1bc360-d822-43a8-b6b7-ff66dc27caf4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatAnthropic\n",
|
||||
"from langchain.prompts import ChatPromptTemplate\n",
|
||||
"from langchain.schema.output_parser import StrOutputParser\n",
|
||||
"from langchain.schema.runnable.passthrough import RunnableAssign\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_chain(retriever):\n",
|
||||
" prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\n",
|
||||
" \"system\",\n",
|
||||
" \"Answer based solely on the retrieved documents below:\\n\\n<Documents>\\n{docs}</Documents>\",\n",
|
||||
" ),\n",
|
||||
" (\"user\", \"{question}\"),\n",
|
||||
" ]\n",
|
||||
" )\n",
|
||||
" llm = ChatAnthropic(model=\"claude-2\")\n",
|
||||
" return (\n",
|
||||
" RunnableAssign({\"docs\": (lambda x: next(iter(x.values()))) | retriever})\n",
|
||||
" | prompt\n",
|
||||
" | llm\n",
|
||||
" | StrOutputParser()\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "935cacd9-e841-4c76-ac16-f3f0cf18df62",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'cold-attachment-88' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/d8e512b7-b63d-4eb5-8d73-d95f7fa7ffc2?eval=true\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Semi-structured Reports at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/f8f24935-cf57-4cb3-a30f-8df303a46962\n",
|
||||
"[------------------------------------------------->] 5/5\n",
|
||||
" Eval quantiles:\n",
|
||||
" inputs.question \\\n",
|
||||
"count 5 \n",
|
||||
"unique 5 \n",
|
||||
"top Analyzing the operating expenses for Q3 2023, ... \n",
|
||||
"freq 1 \n",
|
||||
"mean NaN \n",
|
||||
"std NaN \n",
|
||||
"min NaN \n",
|
||||
"25% NaN \n",
|
||||
"50% NaN \n",
|
||||
"75% NaN \n",
|
||||
"max NaN \n",
|
||||
"\n",
|
||||
" feedback.embedding_cosine_distance feedback.faithfulness \\\n",
|
||||
"count 5.000000 5.0 \n",
|
||||
"unique NaN NaN \n",
|
||||
"top NaN NaN \n",
|
||||
"freq NaN NaN \n",
|
||||
"mean 0.137066 1.0 \n",
|
||||
"std 0.011379 0.0 \n",
|
||||
"min 0.123112 1.0 \n",
|
||||
"25% 0.129089 1.0 \n",
|
||||
"50% 0.137871 1.0 \n",
|
||||
"75% 0.143398 1.0 \n",
|
||||
"max 0.151860 1.0 \n",
|
||||
"\n",
|
||||
" feedback.score_string:accuracy error execution_time \n",
|
||||
"count 5.0 0 5.000000 \n",
|
||||
"unique NaN 0 NaN \n",
|
||||
"top NaN NaN NaN \n",
|
||||
"freq NaN NaN NaN \n",
|
||||
"mean 0.1 NaN 7.940625 \n",
|
||||
"std 0.0 NaN 1.380190 \n",
|
||||
"min 0.1 NaN 6.416387 \n",
|
||||
"25% 0.1 NaN 7.272528 \n",
|
||||
"50% 0.1 NaN 7.324673 \n",
|
||||
"75% 0.1 NaN 8.831243 \n",
|
||||
"max 0.1 NaN 9.858293 \n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.rag import get_eval_config\n",
|
||||
"\n",
|
||||
"client = Client()\n",
|
||||
"RAG_EVALUATION = get_eval_config()\n",
|
||||
"chain = create_chain(retriever)\n",
|
||||
"test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=task.name,\n",
|
||||
" llm_or_chain_factory=chain,\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3b54ef6c-0194-410a-aae9-f30c2097548a",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Example processing the docs\n",
|
||||
"\n",
|
||||
"RAG apps are as good as the information they are able to retrieve. Let's try indexing the tables' summaries to\n",
|
||||
"improve the likelihood that they are retrieved whenever a user asks a relevant question.\n",
|
||||
"\n",
|
||||
"We will use unstructured's `partition_pdf` functionality and generate summaries using an LLM.\n",
|
||||
"\n",
|
||||
"You can define your own indexing pipeline to see how it impacts the downstream performance."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "3378eddb-0a8d-4179-8e9c-54343469eef6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from operator import itemgetter\n",
|
||||
"\n",
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.prompts import ChatPromptTemplate\n",
|
||||
"from langchain.schema.document import Document\n",
|
||||
"from langchain.schema.output_parser import StrOutputParser\n",
|
||||
"from langchain.schema.runnable.passthrough import RunnableAssign\n",
|
||||
"\n",
|
||||
"# Prompt\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\n",
|
||||
" \"system\",\n",
|
||||
" \"You are summarizing semi-structured tables or text in a pdf.\\n\\n```document\\n{doc}\\n```\",\n",
|
||||
" ),\n",
|
||||
" (\"user\", \"Write a concise summary.\"),\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Summary chain\n",
|
||||
"model = ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo-16k\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_doc(x) -> Document:\n",
|
||||
" return Document(\n",
|
||||
" page_content=x[\"output\"],\n",
|
||||
" metadata=x[\"doc\"].metadata,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"summarize_chain = (\n",
|
||||
" {\"doc\": lambda x: x}\n",
|
||||
" | RunnableAssign({\"prompt\": prompt})\n",
|
||||
" | {\n",
|
||||
" \"output\": itemgetter(\"prompt\") | model | StrOutputParser(),\n",
|
||||
" \"doc\": itemgetter(\"doc\"),\n",
|
||||
" }\n",
|
||||
" | create_doc\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "07a2f070-3b5a-4de0-b3da-ddfb6e6f8c2b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"summaries = summarize_chain.batch(\n",
|
||||
" [doc for doc in docs if doc.metadata[\"element_type\"] == \"table\"]\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "22dc0bf8-fa50-4be3-8d23-04f6129548e0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Index the documents and create the retriever. We will re"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 44,
|
||||
"id": "35a1ccf6-2c2f-46f2-838e-5a5bf89515f5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "029921ed3d7c4f389c666583a7192144",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/36 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Indexes the documents with the specified embeddings\n",
|
||||
"retriever_with_summaries = retriever_factory(\n",
|
||||
" embeddings,\n",
|
||||
" docs=docs + summaries,\n",
|
||||
" # Specify a unique transformation name to avoid local cache collisions with other indices.\n",
|
||||
" transformation_name=\"docs-with_summaries\",\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3821e4b0-8e67-418a-840c-470fcde42df0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Evaluate\n",
|
||||
"\n",
|
||||
"We'll evaluate the new chain on the same dataset."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 45,
|
||||
"id": "e6f08c4c-a738-4449-9190-5a4f0b65b99a",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"View the evaluation results for project 'crazy-harmony-39' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/b69d796f-6ba4-4cde-822f-db363cf81f8f?eval=true\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Semi-structured Reports at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/f8f24935-cf57-4cb3-a30f-8df303a46962\n",
|
||||
"[------------------------------------------------->] 5/5\n",
|
||||
" Eval quantiles:\n",
|
||||
" inputs.question \\\n",
|
||||
"count 5 \n",
|
||||
"unique 5 \n",
|
||||
"top Analyzing the operating expenses for Q3 2023, ... \n",
|
||||
"freq 1 \n",
|
||||
"mean NaN \n",
|
||||
"std NaN \n",
|
||||
"min NaN \n",
|
||||
"25% NaN \n",
|
||||
"50% NaN \n",
|
||||
"75% NaN \n",
|
||||
"max NaN \n",
|
||||
"\n",
|
||||
" feedback.score_string:accuracy feedback.faithfulness \\\n",
|
||||
"count 5.000000 5.0 \n",
|
||||
"unique NaN NaN \n",
|
||||
"top NaN NaN \n",
|
||||
"freq NaN NaN \n",
|
||||
"mean 0.720000 1.0 \n",
|
||||
"std 0.408656 0.0 \n",
|
||||
"min 0.100000 1.0 \n",
|
||||
"25% 0.500000 1.0 \n",
|
||||
"50% 1.000000 1.0 \n",
|
||||
"75% 1.000000 1.0 \n",
|
||||
"max 1.000000 1.0 \n",
|
||||
"\n",
|
||||
" feedback.embedding_cosine_distance error execution_time \n",
|
||||
"count 5.000000 0 5.000000 \n",
|
||||
"unique NaN 0 NaN \n",
|
||||
"top NaN NaN NaN \n",
|
||||
"freq NaN NaN NaN \n",
|
||||
"mean 0.069363 NaN 8.659120 \n",
|
||||
"std 0.023270 NaN 2.611724 \n",
|
||||
"min 0.039593 NaN 6.283505 \n",
|
||||
"25% 0.050176 NaN 6.723136 \n",
|
||||
"50% 0.078912 NaN 7.441743 \n",
|
||||
"75% 0.084389 NaN 10.673265 \n",
|
||||
"max 0.093747 NaN 12.173952 \n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"chain_2 = create_chain(retriever_with_summaries)\n",
|
||||
"\n",
|
||||
"test_run_with_summaries = client.run_on_dataset(\n",
|
||||
" dataset_name=task.name,\n",
|
||||
" llm_or_chain_factory=chain_2,\n",
|
||||
" evaluation=RAG_EVALUATION,\n",
|
||||
" verbose=True,\n",
|
||||
")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
-317
@@ -1,317 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b6856d11-40d5-48e5-9eb3-423f479933a1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Semi-structured eval: Chunk size tuning\n",
|
||||
"\n",
|
||||
"`Semi-structured Reports` is a public dataset that contains question-answer pairs from documents with text and tables.\n",
|
||||
"\n",
|
||||
"The question-answer pairs are derived from the tables as well as some of the paragraphs in the docs.\n",
|
||||
"\n",
|
||||
"We evaluation performance of various chunk sizes with RAG. \n",
|
||||
"\n",
|
||||
"## Pre-requisites"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c387b660-967d-4d2f-8c38-af125f7b7a8b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# %pip install -U langchain langsmith langchain_benchmarks\n",
|
||||
"# %pip install --quiet chromadb openai pypdf tiktoken fireworks-ai"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e9e332b1-7da4-47fc-8d9a-4d65fbfc6953",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import getpass\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
|
||||
"env_vars = [\"LANGCHAIN_API_KEY\", \"OPENAI_API_KEY\", \"FIREWORKS_API_KEY\"]\n",
|
||||
"for var in env_vars:\n",
|
||||
" if var not in os.environ:\n",
|
||||
" os.environ[var] = getpass.getpass(prompt=f\"Enter your {var}: \")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b1a19f23-468c-4aeb-a0e9-0765a85f3f0b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Dataset\n",
|
||||
"\n",
|
||||
"Fetch the associated PDFs from remote cache for the dataset so that we can perform ingestion."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "a94d9aa5-acd8-4032-ad8f-f995dec4d13c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks import clone_public_dataset, registry\n",
|
||||
"from langchain_benchmarks.rag.tasks.semi_structured_reports import get_file_names\n",
|
||||
"\n",
|
||||
"# Task\n",
|
||||
"task = registry[\"Semi-structured Reports\"]\n",
|
||||
"\n",
|
||||
"# Files used\n",
|
||||
"paths = list(get_file_names())\n",
|
||||
"files = [str(p) for p in paths]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "12b52285-358c-4752-ad6b-25ffb629e309",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Clone the dataset so that it's available in our LangSmith datasets."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "1ecca7af-c3e7-42d1-97dd-c7d9777207cb",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dataset Semi-structured Reports already exists. Skipping.\n",
|
||||
"You can access the dataset at https://smith.langchain.com/o/1fa8b1f4-fcb9-4072-9aa9-983e35ad61b8/datasets/6549a3a5-1cb9-463f-951d-0166cb9cf45c.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clone_public_dataset(task.dataset_id, dataset_name=task.name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "64f37705-0190-4b7a-9d88-63bfd904fbd9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load and index\n",
|
||||
"\n",
|
||||
"We load each file, split it, embed with `OpenAIEmbeddings`, and create an index with `Chroma` vectorstore."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7eb9e333-77e6-48f9-b221-9bded023b978",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatFireworks, ChatOpenAI\n",
|
||||
"from langchain.document_loaders import PyPDFLoader\n",
|
||||
"from langchain.embeddings import OpenAIEmbeddings\n",
|
||||
"from langchain.prompts import ChatPromptTemplate\n",
|
||||
"from langchain.schema.output_parser import StrOutputParser\n",
|
||||
"from langchain.schema.runnable import RunnablePassthrough\n",
|
||||
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
|
||||
"from langchain.vectorstores import Chroma\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_and_split(file, token_count, split_document=True):\n",
|
||||
" \"\"\"\n",
|
||||
" Load and optionally split PDF files.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" file (str): File path.\n",
|
||||
" token_count (int): Token count for splitting.\n",
|
||||
" split_document (bool): Flag for splitting or returning pages.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" loader = PyPDFLoader(file)\n",
|
||||
" pdf_pages = loader.load()\n",
|
||||
"\n",
|
||||
" if split_document:\n",
|
||||
" text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
|
||||
" chunk_size=token_count, chunk_overlap=50\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" docs = text_splitter.split_documents(pdf_pages)\n",
|
||||
" texts = [d.page_content for d in docs]\n",
|
||||
" else:\n",
|
||||
" texts = [d.page_content for d in pdf_pages]\n",
|
||||
"\n",
|
||||
" print(f\"There are {len(texts)} text elements\")\n",
|
||||
" return texts\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_files(files, token_count, split_document):\n",
|
||||
" \"\"\"\n",
|
||||
" Load files.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" files (list): List of file names.\n",
|
||||
" dir (str): Directory path.\n",
|
||||
" token_count (int): Token count for splitting.\n",
|
||||
" split_document (bool): Flag for splitting documents.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" texts = []\n",
|
||||
" for fi in files:\n",
|
||||
" texts.extend(load_and_split(fi, token_count, split_document))\n",
|
||||
" return texts\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def make_retriever(texts, expt):\n",
|
||||
" \"\"\"\n",
|
||||
" Make vector store.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" texts (list): List of texts.\n",
|
||||
" expt (str): Experiment name.\n",
|
||||
" \"\"\"\n",
|
||||
" vectorstore = Chroma.from_texts(\n",
|
||||
" texts=texts, collection_name=expt, embedding=OpenAIEmbeddings()\n",
|
||||
" )\n",
|
||||
" retriever = vectorstore.as_retriever()\n",
|
||||
" return retriever\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def rag_chain(retriever, llm):\n",
|
||||
" \"\"\"\n",
|
||||
" RAG chain.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" retriever: The retriever to use.\n",
|
||||
" llm: The llm to use.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" # Prompt template\n",
|
||||
" template = \"\"\"Answer the question based only on the following context, which can include text and tables:\n",
|
||||
" {context}\n",
|
||||
" Question: {question}\n",
|
||||
" \"\"\"\n",
|
||||
" prompt = ChatPromptTemplate.from_template(template)\n",
|
||||
"\n",
|
||||
" # LLM\n",
|
||||
" if llm == \"mixtral\":\n",
|
||||
" model = ChatFireworks(\n",
|
||||
" model=\"accounts/fireworks/models/mixtral-8x7b-instruct\", temperature=0\n",
|
||||
" )\n",
|
||||
" else:\n",
|
||||
" model = ChatOpenAI(temperature=0, model=\"gpt-4\")\n",
|
||||
"\n",
|
||||
" # RAG pipeline\n",
|
||||
" chain = (\n",
|
||||
" {\n",
|
||||
" \"context\": retriever | (lambda x: \"\\n\\n\".join([i.page_content for i in x])),\n",
|
||||
" \"question\": RunnablePassthrough(),\n",
|
||||
" }\n",
|
||||
" | prompt\n",
|
||||
" | model\n",
|
||||
" | StrOutputParser()\n",
|
||||
" )\n",
|
||||
" return chain\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Experiment configurations\n",
|
||||
"experiments = [\n",
|
||||
" (None, False, \"page_split-oai\", \"oai\"),\n",
|
||||
" (50, True, \"50_tok_split-oai\", \"oai\"),\n",
|
||||
" (100, True, \"100_tok_split-oai\", \"oai\"),\n",
|
||||
" (250, True, \"250_tok_split-oai\", \"oai\"),\n",
|
||||
" (250, True, \"250_tok_split-mixtral\", \"mixtral\"),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Run\n",
|
||||
"stor_chain = {}\n",
|
||||
"for token_count, split_document, expt, llm in experiments:\n",
|
||||
" texts = load_files(files, token_count, split_document)\n",
|
||||
" retriever = make_retriever(texts, expt)\n",
|
||||
" stor_chain[expt] = rag_chain(retriever, llm)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "29515a91-3cb1-41bd-a2d4-6cf6ce7806c2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Eval\n",
|
||||
"\n",
|
||||
"Run eval onm our dataset, `Semi-structured Reports`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "edd2e7f9-b3f6-4885-bf05-96f1c1758b20",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"from langchain.smith import RunEvalConfig\n",
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"# Config\n",
|
||||
"client = Client()\n",
|
||||
"eval_config = RunEvalConfig(\n",
|
||||
" evaluators=[\"cot_qa\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Experiments\n",
|
||||
"chain_map = {\n",
|
||||
" \"page_split\": stor_chain[\"page_split-oai\"],\n",
|
||||
" \"baseline-50-tok\": stor_chain[\"50_tok_split-oai\"],\n",
|
||||
" \"baseline-100-tok\": stor_chain[\"100_tok_split-oai\"],\n",
|
||||
" \"baseline-250-tok\": stor_chain[\"250_tok_split-oai\"],\n",
|
||||
" \"baseline-250-tok-mixtral\": stor_chain[\"250_tok_split-mixtral\"],\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Run evaluation\n",
|
||||
"run_id = uuid.uuid4().hex[:4]\n",
|
||||
"test_runs = {}\n",
|
||||
"for project_name, chain in chain_map.items():\n",
|
||||
" test_runs[project_name] = client.run_on_dataset(\n",
|
||||
" dataset_name=task.name,\n",
|
||||
" llm_or_chain_factory=lambda: (lambda x: x[\"question\"]) | chain,\n",
|
||||
" evaluation=eval_config,\n",
|
||||
" verbose=True,\n",
|
||||
" project_name=f\"{run_id}-{project_name}\",\n",
|
||||
" project_metadata={\"chain\": project_name},\n",
|
||||
" )"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.16"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
-365
File diff suppressed because one or more lines are too long
-434
@@ -1,434 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7cd0617a-4d00-4c4c-a5df-abc3430e7897",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Semi-structured eval: Multi vector\n",
|
||||
"\n",
|
||||
"`Semi-structured Reports` is a public dataset that contains question-answer pairs from documents with text and tables.\n",
|
||||
"\n",
|
||||
"The question-answer pairs are derived from the tables as well as some of the paragraphs in the docs.\n",
|
||||
"\n",
|
||||
"We evaluation performance using multi-vector retriever for RAG. \n",
|
||||
"\n",
|
||||
"## Pre-requisites"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4edd540d-705f-4042-9ed0-aee42d29f37d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# %pip install -U langchain langsmith langchain_benchmarks\n",
|
||||
"# %pip install --quiet chromadb openai pypdf tiktoken"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "29031433-53db-43bb-ab1a-8ac1721661e8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import getpass\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
|
||||
"env_vars = [\"LANGCHAIN_API_KEY\", \"OPENAI_API_KEY\"]\n",
|
||||
"for var in env_vars:\n",
|
||||
" if var not in os.environ:\n",
|
||||
" os.environ[var] = getpass.getpass(prompt=f\"Enter your {var}: \")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b560e044-f5ac-418b-b3d6-164b423ab23b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Dataset\n",
|
||||
"\n",
|
||||
"Fetch the associated PDFs from remote cache for the dataset so that we can perform ingestion."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "76f8b0e3-693a-4eed-98e7-c0fa9ba02ff9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks import clone_public_dataset, registry\n",
|
||||
"from langchain_benchmarks.rag.tasks.semi_structured_reports import get_file_names\n",
|
||||
"\n",
|
||||
"# Task\n",
|
||||
"task = registry[\"Semi-structured Reports\"]\n",
|
||||
"\n",
|
||||
"# Files used\n",
|
||||
"paths = list(get_file_names())\n",
|
||||
"files = [str(p) for p in paths]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "720016d6-9206-4560-9b12-5881dbcabeb3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Clone the dataset so that it's available in our LangSmith datasets."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1e2309e4-0b35-477b-80a6-d4cb06ca4310",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"clone_public_dataset(task.dataset_id, dataset_name=task.name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fb1db618-05c4-4253-a54b-1c554dd0dc78",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load and index\n",
|
||||
"\n",
|
||||
"We build a retriever that focuses on tables. \n",
|
||||
"\n",
|
||||
"To do this, we use an LLM to scan each page and summarize any tables within the page. \n",
|
||||
"\n",
|
||||
"We then index those summaries for retrieval and store the raw page text containing the table with [multi-vector retriever](https://blog.langchain.dev/semi-structured-multi-modal-rag/). \n",
|
||||
"\n",
|
||||
"Finally, we use [ensemble retriever](https://python.langchain.com/docs/modules/data_connection/retrievers/ensemble) to mix retrieved table chunks with the raw text chunks: \n",
|
||||
"\n",
|
||||
"* Combines the rankings from different retrievers into a single, unified ranking.\n",
|
||||
"* Each retriever provides a list of documents (or search results) ranked based on their relevance to the query.\n",
|
||||
"* The weights represent the relative importance or trust you place in each retriever's results.\n",
|
||||
"* The weights are used to scale the contribution of each retriever to the final combined ranking.\n",
|
||||
"* The RRF method uses the rank of each item in the lists provided by the retrievers.\n",
|
||||
"* The basic idea is to give higher scores to items that are ranked higher (i.e., have a lower rank number) in the lists."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3d14be7d-30c8-4084-afad-3e82c3fbf9e0",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.document_loaders import PyPDFLoader\n",
|
||||
"from langchain.embeddings import OpenAIEmbeddings\n",
|
||||
"from langchain.prompts import ChatPromptTemplate\n",
|
||||
"from langchain.retrievers import EnsembleRetriever\n",
|
||||
"from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
|
||||
"from langchain.schema.document import Document\n",
|
||||
"from langchain.schema.output_parser import StrOutputParser\n",
|
||||
"from langchain.schema.runnable import RunnableLambda, RunnablePassthrough\n",
|
||||
"from langchain.storage import InMemoryStore\n",
|
||||
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
|
||||
"from langchain.vectorstores import Chroma\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def prepare_documents(docs):\n",
|
||||
" \"\"\"\n",
|
||||
" Prepare documents for prompt. Concatenates Document objects (after extracting their page_content)\n",
|
||||
" and strings into a single string, separated by two newlines.\n",
|
||||
"\n",
|
||||
" :param docs: A list of str or Document objects.\n",
|
||||
" :return: A single string containing all documents.\n",
|
||||
" \"\"\"\n",
|
||||
" # Process each document and append it to the list\n",
|
||||
" processed_docs = [\n",
|
||||
" doc.page_content if isinstance(doc, Document) else doc for doc in docs\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" # Join all processed documents into a single string\n",
|
||||
" return \"\\n\\n\".join(processed_docs)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_multi_vector_retriever(vectorstore, text_summaries, texts):\n",
|
||||
" \"\"\"\n",
|
||||
" Create retriever that indexes summaries, but returns raw images or texts\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" # Initialize the storage layer\n",
|
||||
" store = InMemoryStore()\n",
|
||||
" id_key = \"doc_id\"\n",
|
||||
"\n",
|
||||
" # Create the multi-vector retriever\n",
|
||||
" retriever = MultiVectorRetriever(\n",
|
||||
" vectorstore=vectorstore,\n",
|
||||
" docstore=store,\n",
|
||||
" id_key=id_key,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Helper function to add documents to the vectorstore and docstore\n",
|
||||
" def add_documents(retriever, doc_summaries, doc_contents):\n",
|
||||
" doc_ids = [str(uuid.uuid4()) for _ in doc_contents]\n",
|
||||
" summary_docs = [\n",
|
||||
" Document(page_content=s, metadata={id_key: doc_ids[i]})\n",
|
||||
" for i, s in enumerate(doc_summaries)\n",
|
||||
" ]\n",
|
||||
" retriever.vectorstore.add_documents(summary_docs)\n",
|
||||
" retriever.docstore.mset(list(zip(doc_ids, doc_contents)))\n",
|
||||
"\n",
|
||||
" # Add texts, tables, and images\n",
|
||||
" add_documents(retriever, text_summaries, texts)\n",
|
||||
" return retriever\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def generate_doc_summary(file):\n",
|
||||
" \"\"\"\n",
|
||||
" Create a doc summary\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" # Prompt\n",
|
||||
" prompt_text = \"\"\"You are an assistant tasked extracting two attributes \\\n",
|
||||
" from financial documents. (1) Tell me the company that the document is \\\n",
|
||||
" focused on. (2) Look at any tables in the document and tell me the units \\ \n",
|
||||
" of the table. Many table will have '(In thousands)' or '(in millions)' prior \\\n",
|
||||
" to the table text. Provide these two for the document: \\n\\n {document} \"\"\"\n",
|
||||
" prompt = ChatPromptTemplate.from_template(prompt_text)\n",
|
||||
"\n",
|
||||
" # Text summary chain\n",
|
||||
" model = ChatOpenAI(temperature=0, model=\"gpt-4-1106-preview\")\n",
|
||||
" summarize_chain = {\"document\": lambda x: x} | prompt | model | StrOutputParser()\n",
|
||||
"\n",
|
||||
" # Load doc\n",
|
||||
" loader = PyPDFLoader(file)\n",
|
||||
" pdf_pages = loader.load()\n",
|
||||
" texts = [t.page_content for t in pdf_pages]\n",
|
||||
" text_string = \" \".join(texts)\n",
|
||||
" summary = summarize_chain.invoke({\"document\": text_string})\n",
|
||||
" return summary\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def generate_table_summaries(texts):\n",
|
||||
" \"\"\"\n",
|
||||
" Summarize text elements\n",
|
||||
" texts: List of str\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" # Prompt\n",
|
||||
" prompt_text = \"\"\"You are an assistant tasked with summarizing tables within a provided text chunk. \\\n",
|
||||
" If the text chunk contains tables, then give a brief summary of the table and list the row and column \\\n",
|
||||
" names to identify what is captured in the table. Do not sumnmarize quantitative results in the table. \\ \n",
|
||||
" If there is no table present, then just return \"No table\". \\n\\n Text: {element} \"\"\"\n",
|
||||
" prompt = ChatPromptTemplate.from_template(prompt_text)\n",
|
||||
"\n",
|
||||
" # Text summary chain\n",
|
||||
" model = ChatOpenAI(temperature=0, model=\"gpt-4\")\n",
|
||||
" summarize_chain = {\"element\": lambda x: x} | prompt | model | StrOutputParser()\n",
|
||||
"\n",
|
||||
" # Initialize empty summaries\n",
|
||||
" text_summaries = []\n",
|
||||
" text_summaries = summarize_chain.batch(texts, {\"max_concurrency\": 5})\n",
|
||||
"\n",
|
||||
" return text_summaries\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_and_split(file, token_count, split_document=True):\n",
|
||||
" \"\"\"\n",
|
||||
" Load and optionally split PDF files.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" file (str): File path.\n",
|
||||
" token_count (int): Token count for splitting.\n",
|
||||
" split_document (bool): Flag for splitting or returning pages.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" loader = PyPDFLoader(file)\n",
|
||||
" pdf_pages = loader.load()\n",
|
||||
"\n",
|
||||
" if split_document:\n",
|
||||
" text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
|
||||
" chunk_size=token_count, chunk_overlap=50\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" docs = text_splitter.split_documents(pdf_pages)\n",
|
||||
" texts = [d.page_content for d in docs]\n",
|
||||
" else:\n",
|
||||
" texts = [d.page_content for d in pdf_pages]\n",
|
||||
"\n",
|
||||
" print(f\"There are {len(texts)} text elements\")\n",
|
||||
" return texts\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_files(files, token_count, split_document):\n",
|
||||
" \"\"\"\n",
|
||||
" Load files.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" files (list): List of file names.\n",
|
||||
" dir (str): Directory path.\n",
|
||||
" token_count (int): Token count for splitting.\n",
|
||||
" split_document (bool): Flag for splitting documents.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" texts = []\n",
|
||||
" for fi in files:\n",
|
||||
" doc_summary = generate_doc_summary(fi)\n",
|
||||
" texts.extend(load_and_split(fi, token_count, split_document))\n",
|
||||
" return texts, doc_summary\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def rag_chain(retriever):\n",
|
||||
" \"\"\"\n",
|
||||
" RAG chain.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" retriever: The retriever to use.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" # Prompt template\n",
|
||||
" template = \"\"\"Answer the question based only on the following context, which can include text and tables:\n",
|
||||
" {context}\n",
|
||||
" Question: {question}\n",
|
||||
" \"\"\"\n",
|
||||
" prompt = ChatPromptTemplate.from_template(template)\n",
|
||||
"\n",
|
||||
" # LLM\n",
|
||||
" model = ChatOpenAI(temperature=0, model=\"gpt-4\")\n",
|
||||
"\n",
|
||||
" # RAG pipeline\n",
|
||||
" chain = (\n",
|
||||
" {\n",
|
||||
" \"context\": retriever | RunnableLambda(prepare_documents),\n",
|
||||
" \"question\": RunnablePassthrough(),\n",
|
||||
" }\n",
|
||||
" | prompt\n",
|
||||
" | model\n",
|
||||
" | StrOutputParser()\n",
|
||||
" )\n",
|
||||
" return chain\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Experiment configurations\n",
|
||||
"experiments = [\n",
|
||||
" (None, False, \"page_split_multivector\"),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Run\n",
|
||||
"stor_chain = {}\n",
|
||||
"for token_count, split_document, expt in experiments:\n",
|
||||
" # Get texts and doc summary\n",
|
||||
" doc_texts, doc_summary = load_files(files, token_count, split_document)\n",
|
||||
"\n",
|
||||
" # Get table summaries\n",
|
||||
" doc_table_summaries = generate_table_summaries(doc_texts)\n",
|
||||
"\n",
|
||||
" # Add doc summary to table summary to preserve context\n",
|
||||
" doc_text_summaries = [\n",
|
||||
" \"Here is a summary of the doc: \\n\\n\"\n",
|
||||
" + doc_summary\n",
|
||||
" + \"\\n\\n Here is a summary of a table within this doc: \\n\\n\"\n",
|
||||
" + t\n",
|
||||
" for t in doc_table_summaries\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" # The vectorstore to use to index the summaries\n",
|
||||
" vectorstore = Chroma(collection_name=expt, embedding_function=OpenAIEmbeddings())\n",
|
||||
"\n",
|
||||
" # Create our table retriever\n",
|
||||
" table_retriever = create_multi_vector_retriever(\n",
|
||||
" vectorstore, doc_table_summaries, doc_texts\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Create our docs retriever\n",
|
||||
" vectorstore_docs = Chroma.from_texts(\n",
|
||||
" texts=doc_texts, collection_name=expt + \"docs\", embedding=OpenAIEmbeddings()\n",
|
||||
" )\n",
|
||||
" docs_retriever = vectorstore_docs.as_retriever()\n",
|
||||
"\n",
|
||||
" # Initialize ensemble retriever\n",
|
||||
" ensemble_retriever = EnsembleRetriever(\n",
|
||||
" retrievers=[table_retriever, docs_retriever], weights=[0.75, 0.25]\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Chain\n",
|
||||
" stor_chain[expt] = rag_chain(ensemble_retriever)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "77aeb2e2-156d-4a39-be93-4f401f1df455",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Eval\n",
|
||||
"\n",
|
||||
"Run eval onm our dataset, `Semi-structured Reports`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "55fd91b5-6b8e-4bb5-b97a-42ccc5dd53dd",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"from langchain.smith import RunEvalConfig\n",
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"# Config\n",
|
||||
"client = Client()\n",
|
||||
"eval_config = RunEvalConfig(\n",
|
||||
" evaluators=[\"cot_qa\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Experiments\n",
|
||||
"chain_map = {\n",
|
||||
" \"page_split_multivector_emsemble\": stor_chain[\"page_split_multivector\"],\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Run evaluation\n",
|
||||
"run_id = uuid.uuid4().hex[:4]\n",
|
||||
"test_runs = {}\n",
|
||||
"for project_name, chain in chain_map.items():\n",
|
||||
" test_runs[project_name] = client.run_on_dataset(\n",
|
||||
" dataset_name=task.name,\n",
|
||||
" llm_or_chain_factory=lambda: (lambda x: x[\"Question\"]) | chain,\n",
|
||||
" evaluation=eval_config,\n",
|
||||
" verbose=True,\n",
|
||||
" project_name=f\"{run_id}-{project_name}\",\n",
|
||||
" project_metadata={\"chain\": project_name},\n",
|
||||
" )"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.16"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,600 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Running Locally\n",
|
||||
"\n",
|
||||
"The LangChain benchmarks package is best used with LangSmith. You can create a free account [here](https://smith.langchain.com/) and read the [docs here](https://docs.smith.langchain.com/).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"If you are unable to make an account, you can still run these benchmarks locally without an account.\n",
|
||||
"\n",
|
||||
"Below is an example."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a00a1a5f-43ef-4445-a792-8bf6a5f74643",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Prove that we can run without LangSmith\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"_ = [\n",
|
||||
" os.environ.pop(key)\n",
|
||||
" for key in list(os.environ.keys())\n",
|
||||
" if key.startswith(\"LANGCHAIN_\")\n",
|
||||
"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>Multiverse Math </td></tr>\n",
|
||||
"<tr><td>Type </td><td>ToolUsageTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/594f9f60-30a0-49bf-b075-f44beabf546a/d\" target=\"_blank\" rel=\"noopener\">594f9f60-30a0-49bf-b075-f44beabf546a</a></td></tr>\n",
|
||||
"<tr><td>Description</td><td>An environment that contains a few basic math operations, but with altered results.\n",
|
||||
"\n",
|
||||
"For example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"ToolUsageTask(name='Multiverse Math', dataset_id='https://smith.langchain.com/public/594f9f60-30a0-49bf-b075-f44beabf546a/d', description='An environment that contains a few basic math operations, but with altered results.\\n\\nFor example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\\n\\nThe objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math.\\n', create_environment=<function get_environment at 0x137b70360>, instructions='You are requested to solve math questions in an alternate mathematical universe. The operations have been altered to yield different results than expected. Do not guess the answer or rely on your innate knowledge of math. Use the provided tools to answer the question. While associativity and commutativity apply, distributivity does not. Answer the question using the fewest possible tools. Only include the numeric response without any clarifications.', eval_params={'output_evaluation': 'qa_math'})"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_benchmarks import registry\n",
|
||||
"\n",
|
||||
"task = registry[\"Multiverse Math\"]\n",
|
||||
"task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3821e4b0-8e67-418a-840c-470fcde42df0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Eval\n",
|
||||
"\n",
|
||||
"Let's evaluate an agent now. Nothing will be saved to langsmith, so be sure to save the test results to your file system if you want to use them later."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "fb32763c-79ab-426a-8fc6-bf8ebb0dd432",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "bb6a27e067fa4887beaa78a28d8d431d",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Running Evaluation: 0%| | 0/10 [00:00<?, ?example/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<h3>Experiment Results:</h3>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>inputs.question</th>\n",
|
||||
" <th>outputs.input</th>\n",
|
||||
" <th>outputs.output</th>\n",
|
||||
" <th>outputs.intermediate_steps</th>\n",
|
||||
" <th>feedback.Intermediate steps correctness</th>\n",
|
||||
" <th>feedback.# steps / # expected steps</th>\n",
|
||||
" <th>feedback.correctness</th>\n",
|
||||
" <th>error</th>\n",
|
||||
" <th>execution_time</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>count</th>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>10.0</td>\n",
|
||||
" <td>10.0</td>\n",
|
||||
" <td>10.0</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>10.000000</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>unique</th>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>top</th>\n",
|
||||
" <td>multiply the result of (log of 100 to base 10)...</td>\n",
|
||||
" <td>multiply the result of (log of 100 to base 10)...</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>[]</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>freq</th>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>10</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>mean</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>1.453172</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>std</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.496547</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>min</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.763208</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>25%</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.963885</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>50%</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>1.593439</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>75%</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>1.870549</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>max</th>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>NaN</td>\n",
|
||||
" <td>1.957470</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" inputs.question \\\n",
|
||||
"count 10 \n",
|
||||
"unique 10 \n",
|
||||
"top multiply the result of (log of 100 to base 10)... \n",
|
||||
"freq 1 \n",
|
||||
"mean NaN \n",
|
||||
"std NaN \n",
|
||||
"min NaN \n",
|
||||
"25% NaN \n",
|
||||
"50% NaN \n",
|
||||
"75% NaN \n",
|
||||
"max NaN \n",
|
||||
"\n",
|
||||
" outputs.input outputs.output \\\n",
|
||||
"count 10 10 \n",
|
||||
"unique 10 1 \n",
|
||||
"top multiply the result of (log of 100 to base 10)... \n",
|
||||
"freq 1 10 \n",
|
||||
"mean NaN NaN \n",
|
||||
"std NaN NaN \n",
|
||||
"min NaN NaN \n",
|
||||
"25% NaN NaN \n",
|
||||
"50% NaN NaN \n",
|
||||
"75% NaN NaN \n",
|
||||
"max NaN NaN \n",
|
||||
"\n",
|
||||
" outputs.intermediate_steps feedback.Intermediate steps correctness \\\n",
|
||||
"count 10 10.0 \n",
|
||||
"unique 1 NaN \n",
|
||||
"top [] NaN \n",
|
||||
"freq 10 NaN \n",
|
||||
"mean NaN 0.0 \n",
|
||||
"std NaN 0.0 \n",
|
||||
"min NaN 0.0 \n",
|
||||
"25% NaN 0.0 \n",
|
||||
"50% NaN 0.0 \n",
|
||||
"75% NaN 0.0 \n",
|
||||
"max NaN 0.0 \n",
|
||||
"\n",
|
||||
" feedback.# steps / # expected steps feedback.correctness error \\\n",
|
||||
"count 10.0 10.0 0 \n",
|
||||
"unique NaN NaN 0 \n",
|
||||
"top NaN NaN NaN \n",
|
||||
"freq NaN NaN NaN \n",
|
||||
"mean 0.0 0.0 NaN \n",
|
||||
"std 0.0 0.0 NaN \n",
|
||||
"min 0.0 0.0 NaN \n",
|
||||
"25% 0.0 0.0 NaN \n",
|
||||
"50% 0.0 0.0 NaN \n",
|
||||
"75% 0.0 0.0 NaN \n",
|
||||
"max 0.0 0.0 NaN \n",
|
||||
"\n",
|
||||
" execution_time \n",
|
||||
"count 10.000000 \n",
|
||||
"unique NaN \n",
|
||||
"top NaN \n",
|
||||
"freq NaN \n",
|
||||
"mean 1.453172 \n",
|
||||
"std 0.496547 \n",
|
||||
"min 0.763208 \n",
|
||||
"25% 0.963885 \n",
|
||||
"50% 1.593439 \n",
|
||||
"75% 1.870549 \n",
|
||||
"max 1.957470 "
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.tool_usage import agents, get_eval_config\n",
|
||||
"from langchain_benchmarks.utils import run_without_langsmith\n",
|
||||
"\n",
|
||||
"experiment_uuid = uuid.uuid4().hex[:4]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models = [\"gpt-3.5-turbo-1106\"]\n",
|
||||
"\n",
|
||||
"for model in models:\n",
|
||||
" print()\n",
|
||||
" eval_config = get_eval_config(output_evaluation=\"qa_math\")\n",
|
||||
" agent_factory = agents.OpenAIAgentFactory(task, model=model)\n",
|
||||
" test_run = run_without_langsmith(\n",
|
||||
" # This will clone the dataset locally if not already there\n",
|
||||
" path_or_token_id=task.dataset_id,\n",
|
||||
" llm_or_chain_factory=agent_factory,\n",
|
||||
" evaluation=eval_config,\n",
|
||||
" verbose=True,\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "da3015b0-61b2-4748-ab0f-a0239bb74d58",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>inputs.question</th>\n",
|
||||
" <th>outputs.input</th>\n",
|
||||
" <th>outputs.output</th>\n",
|
||||
" <th>outputs.intermediate_steps</th>\n",
|
||||
" <th>feedback.Intermediate steps correctness</th>\n",
|
||||
" <th>feedback.# steps / # expected steps</th>\n",
|
||||
" <th>feedback.correctness</th>\n",
|
||||
" <th>error</th>\n",
|
||||
" <th>execution_time</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>27c44572-6c67-4129-a95a-fe1509c350be</th>\n",
|
||||
" <td>multiply the result of (log of 100 to base 10)...</td>\n",
|
||||
" <td>multiply the result of (log of 100 to base 10)...</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>[]</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" <td>0.763208</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2a20a13d-050e-4a16-84ff-22d9582f1449</th>\n",
|
||||
" <td>after calculating the sin of 1.5 radians, divi...</td>\n",
|
||||
" <td>after calculating the sin of 1.5 radians, divi...</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>[]</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" <td>1.413695</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>67867526-791a-452f-b534-ef2c1f5efd20</th>\n",
|
||||
" <td>ecoli divides every 20 minutes. How many cells...</td>\n",
|
||||
" <td>ecoli divides every 20 minutes. How many cells...</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>[]</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" <td>1.773183</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4ac33c1a-62f0-4da4-9455-07b582f6ff52</th>\n",
|
||||
" <td>calculate 101 to the power of 0.5 to 4 digits ...</td>\n",
|
||||
" <td>calculate 101 to the power of 0.5 to 4 digits ...</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>[]</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" <td>1.819677</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2e82a924-8382-425e-8738-daa2d912e9fe</th>\n",
|
||||
" <td>convert 15 degrees to radians</td>\n",
|
||||
" <td>convert 15 degrees to radians</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>[]</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>0.0</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" <td>1.957470</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" inputs.question \\\n",
|
||||
"27c44572-6c67-4129-a95a-fe1509c350be multiply the result of (log of 100 to base 10)... \n",
|
||||
"2a20a13d-050e-4a16-84ff-22d9582f1449 after calculating the sin of 1.5 radians, divi... \n",
|
||||
"67867526-791a-452f-b534-ef2c1f5efd20 ecoli divides every 20 minutes. How many cells... \n",
|
||||
"4ac33c1a-62f0-4da4-9455-07b582f6ff52 calculate 101 to the power of 0.5 to 4 digits ... \n",
|
||||
"2e82a924-8382-425e-8738-daa2d912e9fe convert 15 degrees to radians \n",
|
||||
"\n",
|
||||
" outputs.input \\\n",
|
||||
"27c44572-6c67-4129-a95a-fe1509c350be multiply the result of (log of 100 to base 10)... \n",
|
||||
"2a20a13d-050e-4a16-84ff-22d9582f1449 after calculating the sin of 1.5 radians, divi... \n",
|
||||
"67867526-791a-452f-b534-ef2c1f5efd20 ecoli divides every 20 minutes. How many cells... \n",
|
||||
"4ac33c1a-62f0-4da4-9455-07b582f6ff52 calculate 101 to the power of 0.5 to 4 digits ... \n",
|
||||
"2e82a924-8382-425e-8738-daa2d912e9fe convert 15 degrees to radians \n",
|
||||
"\n",
|
||||
" outputs.output \\\n",
|
||||
"27c44572-6c67-4129-a95a-fe1509c350be \n",
|
||||
"2a20a13d-050e-4a16-84ff-22d9582f1449 \n",
|
||||
"67867526-791a-452f-b534-ef2c1f5efd20 \n",
|
||||
"4ac33c1a-62f0-4da4-9455-07b582f6ff52 \n",
|
||||
"2e82a924-8382-425e-8738-daa2d912e9fe \n",
|
||||
"\n",
|
||||
" outputs.intermediate_steps \\\n",
|
||||
"27c44572-6c67-4129-a95a-fe1509c350be [] \n",
|
||||
"2a20a13d-050e-4a16-84ff-22d9582f1449 [] \n",
|
||||
"67867526-791a-452f-b534-ef2c1f5efd20 [] \n",
|
||||
"4ac33c1a-62f0-4da4-9455-07b582f6ff52 [] \n",
|
||||
"2e82a924-8382-425e-8738-daa2d912e9fe [] \n",
|
||||
"\n",
|
||||
" feedback.Intermediate steps correctness \\\n",
|
||||
"27c44572-6c67-4129-a95a-fe1509c350be 0 \n",
|
||||
"2a20a13d-050e-4a16-84ff-22d9582f1449 0 \n",
|
||||
"67867526-791a-452f-b534-ef2c1f5efd20 0 \n",
|
||||
"4ac33c1a-62f0-4da4-9455-07b582f6ff52 0 \n",
|
||||
"2e82a924-8382-425e-8738-daa2d912e9fe 0 \n",
|
||||
"\n",
|
||||
" feedback.# steps / # expected steps \\\n",
|
||||
"27c44572-6c67-4129-a95a-fe1509c350be 0.0 \n",
|
||||
"2a20a13d-050e-4a16-84ff-22d9582f1449 0.0 \n",
|
||||
"67867526-791a-452f-b534-ef2c1f5efd20 0.0 \n",
|
||||
"4ac33c1a-62f0-4da4-9455-07b582f6ff52 0.0 \n",
|
||||
"2e82a924-8382-425e-8738-daa2d912e9fe 0.0 \n",
|
||||
"\n",
|
||||
" feedback.correctness error \\\n",
|
||||
"27c44572-6c67-4129-a95a-fe1509c350be 0 None \n",
|
||||
"2a20a13d-050e-4a16-84ff-22d9582f1449 0 None \n",
|
||||
"67867526-791a-452f-b534-ef2c1f5efd20 0 None \n",
|
||||
"4ac33c1a-62f0-4da4-9455-07b582f6ff52 0 None \n",
|
||||
"2e82a924-8382-425e-8738-daa2d912e9fe 0 None \n",
|
||||
"\n",
|
||||
" execution_time \n",
|
||||
"27c44572-6c67-4129-a95a-fe1509c350be 0.763208 \n",
|
||||
"2a20a13d-050e-4a16-84ff-22d9582f1449 1.413695 \n",
|
||||
"67867526-791a-452f-b534-ef2c1f5efd20 1.773183 \n",
|
||||
"4ac33c1a-62f0-4da4-9455-07b582f6ff52 1.819677 \n",
|
||||
"2e82a924-8382-425e-8738-daa2d912e9fe 1.957470 "
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# You can interact with the object directly or as a flattened dataframe\n",
|
||||
"df = test_run.to_dataframe()\n",
|
||||
"df.head()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "1bf4ea77-147f-4687-a2c6-7528a6eba08d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df.to_csv(\"output.csv\", index=False)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,529 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6aae613b-6adb-4e6f-bae7-4974358e07aa",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Benchmark All Tasks\n",
|
||||
"\n",
|
||||
"Let's benchmark against all tool usage tasks. \n",
|
||||
"\n",
|
||||
"Expand the `test` list to benchmark with different models and agent architectures.\n",
|
||||
"\n",
|
||||
"Note that this requires `langsmith>=0.0.72` to run the viz parts at the end."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4525d100-b612-4118-af91-6bdc4aa3fb38",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Set Up\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Credentials\n",
|
||||
"\n",
|
||||
"First, let's set up the models to be tested and the credentials."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "387c494b-ad7e-452e-8d11-0d5d28db855c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"from getpass import getpass\n",
|
||||
"\n",
|
||||
"# This is just the default list below\n",
|
||||
"required_env_vars = [\n",
|
||||
" \"LANGCHAIN_API_KEY\",\n",
|
||||
" \"ANTHROPIC_API_KEY\",\n",
|
||||
" \"OPENAI_API_KEY\",\n",
|
||||
" \"MISTRAL_API_KEY\",\n",
|
||||
"]\n",
|
||||
"for var in required_env_vars:\n",
|
||||
" if var not in os.environ:\n",
|
||||
" os.environ[var] = getpass(f\"Provide the required {var}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d45e54ab-ebbe-4b9a-a596-facae66e1ced",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Instantiate Models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "d3a4e40a-5850-4a0b-b9af-36e9c8b55e8b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_anthropic import ChatAnthropic\n",
|
||||
"from langchain_core.tools import tool\n",
|
||||
"from langchain_google_vertexai import ChatVertexAI\n",
|
||||
"from langchain_mistralai import ChatMistralAI\n",
|
||||
"from langchain_openai import ChatOpenAI\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.tool_usage.agents import StandardAgentFactory\n",
|
||||
"\n",
|
||||
"tests = [\n",
|
||||
" (\n",
|
||||
" \"gemini-1.0-pro-002\",\n",
|
||||
" ChatVertexAI(model_name=\"gemini-1.0-pro-002\", temperature=0),\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" \"gemini-1.5-pro-preview-0409\",\n",
|
||||
" ChatVertexAI(model_name=\"gemini-1.5-pro-preview-0409\", temperature=0),\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" \"open-mixtral-8x22b-2404\",\n",
|
||||
" ChatMistralAI(model=\"open-mixtral-8x22b-2404\", temperature=0),\n",
|
||||
" ),\n",
|
||||
" (\"mistral-large-2402\", ChatMistralAI(model=\"mistral-large-2402\", temperature=0)),\n",
|
||||
" (\n",
|
||||
" \"claude-3-opus-20240229\",\n",
|
||||
" ChatAnthropic(model=\"claude-3-opus-20240229\", temperature=0),\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" \"claude-3-haiku-20240307\",\n",
|
||||
" ChatAnthropic(model=\"claude-3-haiku-20240307\", temperature=0),\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" \"claude-3-sonnet-20240229\",\n",
|
||||
" ChatAnthropic(model=\"claude-3-sonnet-20240229\", temperature=0),\n",
|
||||
" ),\n",
|
||||
" (\"gpt-3.5-turbo-0125\", ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)),\n",
|
||||
" (\n",
|
||||
" \"gpt-4-turbo-2024-04-09\",\n",
|
||||
" ChatOpenAI(model=\"gpt-4-turbo-2024-04-09\", temperature=0),\n",
|
||||
" ),\n",
|
||||
"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6308c18a-209c-44f8-b762-7a07851101f2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Set up the experiment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "9e152e2e-1fb1-4918-9a53-0744c0ef0035",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import datetime\n",
|
||||
"\n",
|
||||
"from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks import (\n",
|
||||
" __version__,\n",
|
||||
" clone_public_dataset,\n",
|
||||
" model_registry,\n",
|
||||
" registry,\n",
|
||||
")\n",
|
||||
"from langchain_benchmarks.rate_limiting import RateLimiter"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "28e4664d-00a1-473b-ae83-f2435962971a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create prompts for the agents\n",
|
||||
"# Using two prompts because some chat models do not support SystemMessage.\n",
|
||||
"without_system_message_prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\n",
|
||||
" \"human\",\n",
|
||||
" \"{instructions}\\n{question}\",\n",
|
||||
" ), # Populated from task.instructions automatically\n",
|
||||
" MessagesPlaceholder(\"agent_scratchpad\"), # Workspace for the agent\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"with_system_message_prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", \"{instructions}\"),\n",
|
||||
" (\"human\", \"{question}\"), # Populated from task.instructions automatically\n",
|
||||
" MessagesPlaceholder(\"agent_scratchpad\"), # Workspace for the agent\n",
|
||||
" ]\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a165f3a1-4e70-4caa-b082-78d4e0c56410",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Generate an experiment id.\n",
|
||||
"\n",
|
||||
"We can tag our runs with this experiment ID and pull data from LangSmith using this experiment ID."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "066d7695-416c-4faf-8c33-c40e5f136672",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"experiment_uuid = \"sky25\" # Or generate ranom using uuid.uuid4().hex[:4]\n",
|
||||
"# experiment_uuid = uuid.uuid4().hex[:4]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d125aad7-cac7-4ec7-9c18-98defe9d2236",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Run"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "03c4c45e-88a6-4c96-ba5d-cfaf03905789",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"client = Client() # Launch langsmith client for cloning datasets\n",
|
||||
"today = datetime.date.today().isoformat()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"for task in registry.tasks:\n",
|
||||
" if task.type != \"ToolUsageTask\":\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" # This is a small test dataset that can be used to verify\n",
|
||||
" # that everything is set up correctly prior to running over\n",
|
||||
" # all results. We may remove it in the future.\n",
|
||||
" if task.name == \"Multiverse Math (Tiny)\":\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" dataset_name = task.name + f\" ({today})\"\n",
|
||||
" clone_public_dataset(task.dataset_id, dataset_name=dataset_name)\n",
|
||||
"\n",
|
||||
" for model_name, model in tests:\n",
|
||||
" if model_name.startswith(\"gemini\"):\n",
|
||||
" # google models don't use system prompt\n",
|
||||
" prompt = without_system_message_prompt\n",
|
||||
" rate_limiter = RateLimiter(requests_per_second=0.1)\n",
|
||||
" else:\n",
|
||||
" prompt = with_system_message_prompt\n",
|
||||
" rate_limiter = RateLimiter(requests_per_second=1)\n",
|
||||
" print()\n",
|
||||
" print(f\"Benchmarking {task.name} with model: {model_name}\")\n",
|
||||
" eval_config = task.get_eval_config()\n",
|
||||
"\n",
|
||||
" agent_factory = StandardAgentFactory(\n",
|
||||
" task, model, prompt, rate_limiter=rate_limiter\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" client.run_on_dataset(\n",
|
||||
" dataset_name=dataset_name,\n",
|
||||
" llm_or_chain_factory=agent_factory,\n",
|
||||
" evaluation=eval_config,\n",
|
||||
" verbose=False,\n",
|
||||
" project_name=f\"{model_name}-{task.name}-{today}-{experiment_uuid}\",\n",
|
||||
" concurrency_level=5,\n",
|
||||
" project_metadata={\n",
|
||||
" \"model\": model_name,\n",
|
||||
" \"id\": experiment_uuid,\n",
|
||||
" \"task\": task.name,\n",
|
||||
" \"date\": today,\n",
|
||||
" \"langchain_benchmarks_version\": __version__,\n",
|
||||
" },\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "54e7999f-e8ab-45a6-88a9-0ae76f3d24cf",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Inspect\n",
|
||||
"\n",
|
||||
"Note that if the queue is under significant load, you may want to wait before running the following to ensure all runs are in the DB and all stats are correctly computed."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "598b92f0-7d64-4731-b294-05948d4db562",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install --quiet -U pandas"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "7818572a-a5fb-4153-bbe0-6f9e90813a22",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"from langsmith.client import Client"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e7890951-ffde-4706-95e5-ae3e9bf0e8a6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's fetch all the data that has the same experiment ID and place it in a dataframe."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"id": "44822aa4-8c4e-46be-8126-b79a9acdf8e1",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"experiment_ids = [\"sky25\"]\n",
|
||||
"dataset_names = [\n",
|
||||
" \"Tool Usage - Typewriter (1 tool)\",\n",
|
||||
" \"Tool Usage - Typewriter (26 tools)\",\n",
|
||||
" \"Tool Usage - Relational Data\",\n",
|
||||
" \"Multiverse Math\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"client = Client()\n",
|
||||
"projects = []\n",
|
||||
"for dataset_name in dataset_names:\n",
|
||||
" dataset_name_ = dataset_name + f\" ({today})\"\n",
|
||||
" for project in client.list_projects(reference_dataset_name=dataset_name_):\n",
|
||||
" if (\n",
|
||||
" project.metadata.get(\"id\") in experiment_ids\n",
|
||||
" and project.end_time is not None\n",
|
||||
" ):\n",
|
||||
" projects.append(project)\n",
|
||||
"\n",
|
||||
"dfs = []\n",
|
||||
"keys = set()\n",
|
||||
"for project in projects:\n",
|
||||
" # Temporary way to get tag information\n",
|
||||
" try:\n",
|
||||
" test_results = client.get_test_results(project_name=project.name)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e, project.run_count)\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" for k, v in project.metadata.items():\n",
|
||||
" test_results[k] = v\n",
|
||||
" keys.update(test_results.columns)\n",
|
||||
" dfs.append(test_results)\n",
|
||||
"for df in dfs:\n",
|
||||
" missing = list(keys - set(df.columns))\n",
|
||||
" for key in missing:\n",
|
||||
" df[key] = None\n",
|
||||
"df = pd.concat(dfs)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9065b7a0-d514-49f7-9d79-67181c41f56d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Compute a standardized \"correct\" column. It uses \"Correct Final State\" for tool usage tasks, and \"correctness (which is based on output) for the other tasks."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 26,
|
||||
"id": "b3c0466a-25f4-44d7-bd2a-20da51461994",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"correct = []\n",
|
||||
"\n",
|
||||
"for r in df.to_dict(orient=\"records\"):\n",
|
||||
" if \"Typewriter\" in r[\"task\"]:\n",
|
||||
" correct.append(r[\"feedback.correct final state\"])\n",
|
||||
" else:\n",
|
||||
" correct.append(r[\"feedback.correctness\"])\n",
|
||||
"\n",
|
||||
"df[\"correct\"] = correct\n",
|
||||
"df[\"correct\"].fillna(0, inplace=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "270b8ae9-c84b-4ebc-88ab-fa0ac5e28a57",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Compute some statistics. We're using estimating standard error of the mean assuming a bernoulli process."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"id": "c59d080c-d3ac-43c3-a527-9961913db2ba",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"num_correct = df.groupby([\"model\", \"task\"])[\"correct\"].sum().to_frame(\"num_correct\")\n",
|
||||
"total = df.groupby([\"task\", \"model\"]).size().to_frame(\"total\")\n",
|
||||
"stats_df = total.join(num_correct)\n",
|
||||
"stats_df[\"% correct\"] = stats_df[\"num_correct\"] / stats_df[\"total\"]\n",
|
||||
"stats_df[\"error\"] = np.sqrt(\n",
|
||||
" stats_df[\"% correct\"] * (1 - stats_df[\"% correct\"]) / stats_df[\"total\"]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"tasks = [\n",
|
||||
" \"Tool Usage - Typewriter (1 tool)\",\n",
|
||||
" \"Tool Usage - Typewriter (26 tools)\",\n",
|
||||
" \"Multiverse Math\",\n",
|
||||
" \"Tool Usage - Relational Data\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"stats_df = stats_df.reset_index()\n",
|
||||
"models = stats_df[\"model\"].unique()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 28,
|
||||
"id": "bdbd6005-906a-42fd-af05-b4f27e2c3c51",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"array(['claude-3-haiku-20240307', 'claude-3-opus-20240229',\n",
|
||||
" 'claude-3-sonnet-20240229', 'gemini-1.0-pro-002',\n",
|
||||
" 'gemini-1.5-pro-preview-0409', 'gpt-3.5-turbo-0125',\n",
|
||||
" 'gpt-4-turbo-2024-04-09', 'mistral-large-2402',\n",
|
||||
" 'open-mixtral-8x22b-2404'], dtype=object)"
|
||||
]
|
||||
},
|
||||
"execution_count": 28,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1d9f79af-128c-4e2e-8c1e-807e397b9791",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"Plot the result"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "69df66a1-960c-40a3-abc8-58b503fceda5",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from itertools import product\n",
|
||||
"\n",
|
||||
"x = np.arange(len(tasks)) # the label locations\n",
|
||||
"width = 0.06 # the width of the bars\n",
|
||||
"multiplier = 1.1\n",
|
||||
"\n",
|
||||
"fig, ax = plt.subplots(layout=\"constrained\", figsize=(20, 4))\n",
|
||||
"colormap = plt.get_cmap(\"Set3\").colors\n",
|
||||
"idx = 0\n",
|
||||
"for model in models:\n",
|
||||
" try:\n",
|
||||
" results = stats_df.set_index(\"model\").loc[model]\n",
|
||||
" except:\n",
|
||||
" continue\n",
|
||||
" if len(results) == 0:\n",
|
||||
" continue\n",
|
||||
" color = colormap[idx]\n",
|
||||
" idx += 1\n",
|
||||
"\n",
|
||||
" results = results.set_index(\"task\").loc[tasks]\n",
|
||||
" measurement = results[\"% correct\"]\n",
|
||||
"\n",
|
||||
" values = [round(m, 2) for m in measurement]\n",
|
||||
"\n",
|
||||
" offset = width * multiplier * 1.4\n",
|
||||
" rects = ax.bar(\n",
|
||||
" x + offset,\n",
|
||||
" values,\n",
|
||||
" width,\n",
|
||||
" label=f\"{model}\",\n",
|
||||
" yerr=results[\"error\"],\n",
|
||||
" color=color,\n",
|
||||
" )\n",
|
||||
" ax.bar_label(rects, padding=3)\n",
|
||||
" multiplier += 1\n",
|
||||
"\n",
|
||||
"# Add some text for labels, title and custom x-axis tick labels, etc.\n",
|
||||
"ax.set_ylabel(\"% Questions Answered Correctly\")\n",
|
||||
"ax.set_title(\"Tool Usage Performance\")\n",
|
||||
"ax.set_xticks(x + width + 0.3, tasks)\n",
|
||||
"ax.legend(\n",
|
||||
" loc=\"center left\", ncols=1, bbox_to_anchor=(1.0, 0.5), frameon=False, title=\"Model\"\n",
|
||||
")\n",
|
||||
"ax.set_ylim(0, 1.10)\n",
|
||||
"plt.savefig(\"overall_perf.png\", dpi=300, bbox_inches=\"tight\")\n",
|
||||
"plt.show()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,814 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1c9df2ed-3496-45c6-8b1b-e12776a02a0f",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Introduction\n",
|
||||
"\n",
|
||||
"Tool Usage tasks are designed to evaluate how well an agent can use tools to accomplish an objective.\n",
|
||||
"\n",
|
||||
"Each task defines an environment in which the agent operates. The environment consists of a set of tools and a way to read the state of the environment (more on that below).\n",
|
||||
"\n",
|
||||
"The tasks allow you to stress test the agent in different ways:\n",
|
||||
"\n",
|
||||
"* Can the agent use a single tool effectively?\n",
|
||||
"* Can the agent use more than 10 tools effectively?\n",
|
||||
"* Can the agent correctly incorporate information returned by the tool (and ignore internal knowledge)?\n",
|
||||
"\n",
|
||||
"To help in this evaluation, each task is associated with a LangSmith dataset that includes input/output examples of varying difficulties.\n",
|
||||
"\n",
|
||||
"## Schema\n",
|
||||
"\n",
|
||||
"To make it possible to evaluate different agent implementations, we're using a standardized schema, we'll illustrate it with the following example taken from tool usage.\n",
|
||||
"\n",
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"Each task corresponds to a LangSmith dataset with the following schema:\n",
|
||||
"\n",
|
||||
"Inputs:\n",
|
||||
"\n",
|
||||
"| name | type | meaning |\n",
|
||||
"| ----------- | ----------- | -----------------------|\n",
|
||||
"| question | str | the user question |\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Outputs:\n",
|
||||
"\n",
|
||||
"| name | type | meaning |\n",
|
||||
"| ------------- | --------------- | ------------------------------------------------------|\n",
|
||||
"| reference | str | the expected answer |\n",
|
||||
"| expected_steps| List[str] | the list of tools that should be invoked |\n",
|
||||
"| order_matters | bool | whether the tools should be invoked in the specific order |\n",
|
||||
"| state | Optional[Any] | the state of the system after the agent has taken its actions |\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Here's an [example](https://smith.langchain.com/public/1d89f4b3-5f73-48cf-a127-2fdeb22f6d84/d/e82a0faf-00b9-40a5-a0e3-9723d923e58e/e) contains the following keys/values:\n",
|
||||
"\n",
|
||||
"```json\n",
|
||||
"{\n",
|
||||
" \"input\": {\"question\": \"weather in LA right now?\"},\n",
|
||||
" \"output\": {\n",
|
||||
" \"reference\": \"Sunny, Temperature: 75°F\",\n",
|
||||
" \"order_matters\": true,\n",
|
||||
" \"expected_steps\": [\n",
|
||||
" \"find_locations_by_name\",\n",
|
||||
" \"get_current_weather_for_location\"\n",
|
||||
" ],\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Agent\n",
|
||||
"\n",
|
||||
"To work with the evaluators provided by LangChain Benchmarks (of course you're free to write your own evaluators!).\n",
|
||||
"\n",
|
||||
"An agent must accept `question` as an input and return:\n",
|
||||
"\n",
|
||||
"```json\n",
|
||||
"{\n",
|
||||
" \"output\": \"It's super sunny. Like 75F\", // the output from the agent\n",
|
||||
" \"intermediate_steps\": [... \"find_locations_by_name\" ...], // list of the intermediate steps taken by the agent (see format in LangChain)\n",
|
||||
" \"state\": .., // Can be anything, this is the state fo the environment after the agent has taken all of its actions (optional key)\n",
|
||||
"}\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f478781d-80ad-44ab-a0f8-1fc3d8c0f14d",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"## Tasks\n",
|
||||
"\n",
|
||||
"You can check an up-to-date list of tool usage tasks in the registry: "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "3b9b82fc-b689-4a25-b718-99ecc2fc6867",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<thead>\n",
|
||||
"<tr><th>Name </th><th>Type </th><th>Dataset ID </th><th>Description </th></tr>\n",
|
||||
"</thead>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Tool Usage - Typewriter (1 tool) </td><td>ToolUsageTask</td><td><a href=\"https://smith.langchain.com/public/59577193-8938-4ccf-92a7-e8a96bcf4f86/d\" target=\"_blank\" rel=\"noopener\">59577193-8938-4ccf-92a7-e8a96bcf4f86</a></td><td>Environment with a single tool that accepts a single letter as input, and prints it on a piece of virtual paper.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the ability of the model to use the provided tools to repeat a given input string.\n",
|
||||
"\n",
|
||||
"For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
|
||||
"\n",
|
||||
"The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string. </td></tr>\n",
|
||||
"<tr><td>Tool Usage - Typewriter (26 tools)</td><td>ToolUsageTask</td><td><a href=\"https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d\" target=\"_blank\" rel=\"noopener\">128af05e-aa00-4e3b-a958-d166dd450581</a></td><td>Environment with 26 tools each tool represents a letter of the alphabet.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the model's ability the use tools\n",
|
||||
"for a simple repetition task.\n",
|
||||
"\n",
|
||||
"For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
|
||||
"\n",
|
||||
"The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\n",
|
||||
"\n",
|
||||
"This is a variation of the typer writer task, where 26 parameterless tools are\n",
|
||||
"given instead of a single tool that takes a letter as an argument. </td></tr>\n",
|
||||
"<tr><td>Tool Usage - Relational Data </td><td>ToolUsageTask</td><td><a href=\"https://smith.langchain.com/public/1d89f4b3-5f73-48cf-a127-2fdeb22f6d84/d\" target=\"_blank\" rel=\"noopener\">1d89f4b3-5f73-48cf-a127-2fdeb22f6d84</a></td><td>Environment with fake data about users and their locations and favorite foods.\n",
|
||||
"\n",
|
||||
"The environment provides a set of tools that can be used to query the data.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the ability to use the provided tools to answer questions about relational data.\n",
|
||||
"\n",
|
||||
"The dataset contains 21 examples of varying difficulty. The difficulty is measured by the number of tools that need to be used to answer the question.\n",
|
||||
"\n",
|
||||
"Each example is composed of a question, a reference answer, and information about the sequence in which tools should be used to answer the question.\n",
|
||||
"\n",
|
||||
"Success is measured by the ability to answer the question correctly, and efficiently. </td></tr>\n",
|
||||
"<tr><td>Multiverse Math </td><td>ToolUsageTask</td><td><a href=\"https://smith.langchain.com/public/47ed57bc-e852-4f84-a23e-cce4793864e9/d\" target=\"_blank\" rel=\"noopener\">47ed57bc-e852-4f84-a23e-cce4793864e9</a></td><td>An environment that contains a few basic math operations, but with altered results.\n",
|
||||
"\n",
|
||||
"For example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math.\n",
|
||||
"\n",
|
||||
"This task is associated with 20 test examples. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"Registry(tasks=[ToolUsageTask(name='Tool Usage - Typewriter (1 tool)', dataset_id='https://smith.langchain.com/public/59577193-8938-4ccf-92a7-e8a96bcf4f86/d', description=\"Environment with a single tool that accepts a single letter as input, and prints it on a piece of virtual paper.\\n\\nThe objective of this task is to evaluate the ability of the model to use the provided tools to repeat a given input string.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\", create_environment=<function get_environment at 0x7b3a9f5fad40>, instructions=\"Repeat the given string using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must print the letters 'a', 'b', and 'c' one at a time and in that order. \", eval_params={'output_evaluation': 'none'}), ToolUsageTask(name='Tool Usage - Typewriter (26 tools)', dataset_id='https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d', description=\"Environment with 26 tools each tool represents a letter of the alphabet.\\n\\nThe objective of this task is to evaluate the model's ability the use tools\\nfor a simple repetition task.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\\nThis is a variation of the typer writer task, where 26 parameterless tools are\\ngiven instead of a single tool that takes a letter as an argument.\\n\", create_environment=<function get_environment at 0x7b3a9f5fb240>, instructions=\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\", eval_params={'output_evaluation': 'none'}), ToolUsageTask(name='Tool Usage - Relational Data', dataset_id='https://smith.langchain.com/public/1d89f4b3-5f73-48cf-a127-2fdeb22f6d84/d', description='Environment with fake data about users and their locations and favorite foods.\\n\\nThe environment provides a set of tools that can be used to query the data.\\n\\nThe objective of this task is to evaluate the ability to use the provided tools to answer questions about relational data.\\n\\nThe dataset contains 21 examples of varying difficulty. The difficulty is measured by the number of tools that need to be used to answer the question.\\n\\nEach example is composed of a question, a reference answer, and information about the sequence in which tools should be used to answer the question.\\n\\nSuccess is measured by the ability to answer the question correctly, and efficiently.\\n', create_environment=<function get_environment at 0x7b3a9f5fa840>, instructions=\"Please answer the user's question by using the tools provided. Do not guess the answer. Keep in mind that entities like users,foods and locations have both a name and an ID, which are not the same.\", eval_params={}), ToolUsageTask(name='Multiverse Math', dataset_id='https://smith.langchain.com/public/47ed57bc-e852-4f84-a23e-cce4793864e9/d', description='An environment that contains a few basic math operations, but with altered results.\\n\\nFor example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\\n\\nThe objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math.\\n\\nThis task is associated with 20 test examples.\\n', create_environment=<function get_environment at 0x7b3a9f5fa200>, instructions='You are requested to solve math questions in an alternate mathematical universe. The operations have been altered to yield different results than expected. Do not guess the answer or rely on your innate knowledge of math. Use the provided tools to answer the question. While associativity and commutativity apply, distributivity does not. Answer the question using the fewest possible tools. Only include the numeric response without any clarifications.', eval_params={'output_evaluation': 'qa_math_without_question'})])"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_benchmarks import registry\n",
|
||||
"\n",
|
||||
"registry.filter(Type=\"ToolUsageTask\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9f54cdd3-67f6-43ba-a929-1a6ed1b01296",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's understand what a tool usage task is in a bit more detail"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "7543739b-d212-4249-9b4a-fc406a58c9c7",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>Tool Usage - Typewriter (26 tools) </td></tr>\n",
|
||||
"<tr><td>Type </td><td>ToolUsageTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d\" target=\"_blank\" rel=\"noopener\">128af05e-aa00-4e3b-a958-d166dd450581</a></td></tr>\n",
|
||||
"<tr><td>Description</td><td>Environment with 26 tools each tool represents a letter of the alphabet.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the model's ability the use tools\n",
|
||||
"for a simple repetition task.\n",
|
||||
"\n",
|
||||
"For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
|
||||
"\n",
|
||||
"The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\n",
|
||||
"\n",
|
||||
"This is a variation of the typer writer task, where 26 parameterless tools are\n",
|
||||
"given instead of a single tool that takes a letter as an argument. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"ToolUsageTask(name='Tool Usage - Typewriter (26 tools)', dataset_id='https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d', description=\"Environment with 26 tools each tool represents a letter of the alphabet.\\n\\nThe objective of this task is to evaluate the model's ability the use tools\\nfor a simple repetition task.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\\nThis is a variation of the typer writer task, where 26 parameterless tools are\\ngiven instead of a single tool that takes a letter as an argument.\\n\", create_environment=<function get_environment at 0x7b3a9f5fb240>, instructions=\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\", eval_params={'output_evaluation': 'none'})"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task = registry[\"Tool Usage - Typewriter (26 tools)\"]\n",
|
||||
"task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "393d6c77-8ed0-41d0-a7f5-303356cca4af",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Tool usage tasks are associated with an environment\n",
|
||||
"\n",
|
||||
"---------\n",
|
||||
"```python\n",
|
||||
"\n",
|
||||
"@dataclasses.dataclass(frozen=True)\n",
|
||||
"class ToolUsageEnvironment:\n",
|
||||
" \"\"\"An instance of an environment for tool usage.\"\"\"\n",
|
||||
"\n",
|
||||
" tools: List[BaseTool]\n",
|
||||
" \"\"\"The tools that can be used in the environment.\"\"\"\n",
|
||||
"\n",
|
||||
" read_state: Optional[Callable[[], Any]] = None\n",
|
||||
" \"\"\"A function that returns the current state of the environment.\"\"\"\n",
|
||||
"\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1f7b99c2-82b1-4eb5-87cb-5891d0b16a4b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"--------------\n",
|
||||
"\n",
|
||||
"Here, we'll dig into the typewriter task a bit to explain what the environment state represents.\n",
|
||||
"\n",
|
||||
"The typewrite task has 26 tools each of which prints a letter on a piece of virtual paper"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "f201dbbe-7d92-4bc7-b4b5-ea8901dd2970",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[StructuredTool(name='a', description='a() -> str - Run to Type the letter \"a\".', args_schema=<class 'pydantic.v1.main.aSchema'>, func=<function _create_typing_func.<locals>.func at 0x7b3a9f62c9a0>),\n",
|
||||
" StructuredTool(name='b', description='b() -> str - Run to Type the letter \"b\".', args_schema=<class 'pydantic.v1.main.bSchema'>, func=<function _create_typing_func.<locals>.func at 0x7b3a9f62c5e0>),\n",
|
||||
" StructuredTool(name='c', description='c() -> str - Run to Type the letter \"c\".', args_schema=<class 'pydantic.v1.main.cSchema'>, func=<function _create_typing_func.<locals>.func at 0x7b3a9f62cae0>),\n",
|
||||
" StructuredTool(name='d', description='d() -> str - Run to Type the letter \"d\".', args_schema=<class 'pydantic.v1.main.dSchema'>, func=<function _create_typing_func.<locals>.func at 0x7b3a9f62cb80>)]"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env = task.create_environment()\n",
|
||||
"env.tools[:4]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "b07957ee-ae52-47d4-a4ff-aa99d4d9bdaf",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'OK'"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.tools[0].invoke({}) # Invoke a()\n",
|
||||
"env.tools[0].invoke({}) # invoke a()\n",
|
||||
"env.tools[2].invoke({}) # invoke c()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "40fbb9b6-00f6-4445-b480-00eed6b5b3aa",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'aac'"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.read_state() # Shows the content of the virtual paper"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8d39b9b3-d4da-49bc-b3db-8a4165b1db55",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Create an Agent!\n",
|
||||
"\n",
|
||||
"Now that you know how the test environment works, let's create an agent that we can test!\n",
|
||||
"\n",
|
||||
"Because an agent interacts with the environment via tools and can change the state of the environment during the course of an agent run, what we actually want is the ability to create a fresh agent and a fresh environment for each test run.\n",
|
||||
"\n",
|
||||
"We'll do this using a factory. A factory is just a fancy name in computer science for an object that can create other objects. In this case, we'll have an Agent Factory that we can call and it'll create a fresh agent for us on each call.\n",
|
||||
"\n",
|
||||
"We'll use the StandardAgentFactory which under the hood creates a standard LangChain [tool calling agent](https://python.langchain.com/docs/modules/agents/agent_types/tool_calling/). It can be used with any [Chat Model that support tool calling](https://python.langchain.com/docs/integrations/chat/)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "db65c253-7710-4c7b-b968-0662ec089030",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_anthropic.chat_models import ChatAnthropic\n",
|
||||
"from langchain_core.prompts import ChatPromptTemplate\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.tool_usage.agents import StandardAgentFactory\n",
|
||||
"\n",
|
||||
"model = ChatAnthropic(model=\"claude-3-opus-20240229\", temperature=0)\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", \"{instructions}\"), # Populated from task.instructions automatically\n",
|
||||
" (\n",
|
||||
" \"human\",\n",
|
||||
" \"{question}\",\n",
|
||||
" ), # Each evaluation example is associated with a question\n",
|
||||
" (\"placeholder\", \"{agent_scratchpad}\"), # Space for the agent to do work\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"agent_factory = StandardAgentFactory(task, model, prompt)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5c99a9bd-fa3e-4401-9062-77dbcff30d5c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Here, were the instructions for the task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "8e1f0a3d-fed6-41f7-8825-08787a57ad98",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\""
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task.instructions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "82c9de5d-185b-4776-9ee9-112a2db32139",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's test it out"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "ce67d619-fa99-4c15-bc53-3fb08b40a201",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
|
||||
"\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `a` with `{}`\n",
|
||||
"responded: [{'text': '<thinking>\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\n</thinking>', 'type': 'text'}, {'id': 'toolu_01MQ6oTx2j2uNGCR5LBVeKui', 'input': {}, 'name': 'a', 'type': 'tool_use'}, {'id': 'toolu_01AytT1jvNNR67VodMkhbq7r', 'input': {}, 'name': 'b', 'type': 'tool_use'}, {'id': 'toolu_015VkTYUV5hWcobtduqssi9k', 'input': {}, 'name': 'c', 'type': 'tool_use'}]\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `b` with `{}`\n",
|
||||
"responded: [{'text': '<thinking>\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\n</thinking>', 'type': 'text'}, {'id': 'toolu_01MQ6oTx2j2uNGCR5LBVeKui', 'input': {}, 'name': 'a', 'type': 'tool_use'}, {'id': 'toolu_01AytT1jvNNR67VodMkhbq7r', 'input': {}, 'name': 'b', 'type': 'tool_use'}, {'id': 'toolu_015VkTYUV5hWcobtduqssi9k', 'input': {}, 'name': 'c', 'type': 'tool_use'}]\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[33;1m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `c` with `{}`\n",
|
||||
"responded: [{'text': '<thinking>\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\n</thinking>', 'type': 'text'}, {'id': 'toolu_01MQ6oTx2j2uNGCR5LBVeKui', 'input': {}, 'name': 'a', 'type': 'tool_use'}, {'id': 'toolu_01AytT1jvNNR67VodMkhbq7r', 'input': {}, 'name': 'b', 'type': 'tool_use'}, {'id': 'toolu_015VkTYUV5hWcobtduqssi9k', 'input': {}, 'name': 'c', 'type': 'tool_use'}]\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[38;5;200m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3m[]\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain import globals\n",
|
||||
"\n",
|
||||
"globals.set_verbose(True)\n",
|
||||
"agent = agent_factory()\n",
|
||||
"agent.invoke({\"question\": \"abc\"})\n",
|
||||
"globals.set_verbose(False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e3bce984-7c9c-4f6e-a51b-01c3e2b6e00a",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Benchmarking\n",
|
||||
"\n",
|
||||
"How does one evaluate an agent? Given a particular task and input, an agent uses tools to produce an output AND/OR change the state of the environment.\n",
|
||||
"\n",
|
||||
"To evaluate an agent, we can check the following:\n",
|
||||
"\n",
|
||||
"1. Did the agent use the expected tools?\n",
|
||||
"2. Did the agent use the tools in the most effective way; e.g., was the order of tool invocation correct?\n",
|
||||
"3. Did the environment end up in the correct final state after the agent used the tools? (e.g., does my calendar contain all the scheduled meetings?)\n",
|
||||
"4. Did the agent output match the expected reference output?\n",
|
||||
"\n",
|
||||
"Each task is associated with a standard evaluator that does evaluation that's appropriate for the task; for example,\n",
|
||||
"\n",
|
||||
"1. Use an LLM to grade Compare output to reference using an LLM that grades the response.\n",
|
||||
"2. Compare equality of expected_steps to the list of tools in intermediate_steps -- simple list equality\n",
|
||||
"3. Compare the state of the environment against expected state (if present in the dataset and in the agent)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5e9e5817-3b9d-4a1e-8ee8-692d39aa68ca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Each task is associated with its own task specific evaluator!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "c88bd6e1-f77e-4668-a143-096929e897ee",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"RunEvalConfig(evaluators=[], custom_evaluators=[<langchain_benchmarks.tool_usage.evaluators.AgentTrajectoryEvaluator object at 0x7b3a9ea5b110>], batch_evaluators=None, reference_key=None, prediction_key=None, input_key=None, eval_llm=None)"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"eval_config = task.get_eval_config()\n",
|
||||
"eval_config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "044c7f91-9bb3-44b5-802d-f9f444ddeff9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Set up code to run against all tasks"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "0770b442-f96a-4670-a4f7-3093f24fb64b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import datetime\n",
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks import (\n",
|
||||
" __version__,\n",
|
||||
" clone_public_dataset,\n",
|
||||
" model_registry,\n",
|
||||
" registry,\n",
|
||||
")\n",
|
||||
"from langchain_benchmarks.rate_limiting import RateLimiter"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "15cbded4-5ab5-4b9b-9e88-77b24d3b750c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Create an experiment ID. we'll use it to tag our runs, which we can later use to retrieve run data from LangSmith."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "c23208e3-01d1-4e83-9e4a-59544828f6f5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"experiment_id = uuid.uuid4().hex[:]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "83050cfc-f50f-4c63-8257-07e7688a54c4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Run evaluation against all tasks."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b2a3463b-1c9f-494b-bcbd-1dc1760ebf19",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"client = Client() # Launch langsmith client for cloning datasets\n",
|
||||
"today = datetime.date.today().isoformat()\n",
|
||||
"\n",
|
||||
"# You can use an optional rate limiter to rate limit your requests!\n",
|
||||
"rate_limiter = RateLimiter(requests_per_second=1)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Set up 2-tuples of (model name, model instance)\n",
|
||||
"# You can update this list with any model that supports tool calling.\n",
|
||||
"# See list here: https://python.langchain.com/docs/integrations/chat/\n",
|
||||
"tests = [\n",
|
||||
" (\n",
|
||||
" \"claude-3-haiku-20240307\",\n",
|
||||
" ChatAnthropic(model=\"claude-3-haiku-20240307\", temperature=0),\n",
|
||||
" )\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"for task in registry.tasks:\n",
|
||||
" if task.type != \"ToolUsageTask\":\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" dataset_name = task.name + f\" ({today})\"\n",
|
||||
" clone_public_dataset(task.dataset_id, dataset_name=dataset_name)\n",
|
||||
"\n",
|
||||
" for model_name, model in tests:\n",
|
||||
" print()\n",
|
||||
" print(f\"Benchmarking {task.name} with model: {model_name}\")\n",
|
||||
" eval_config = task.get_eval_config()\n",
|
||||
"\n",
|
||||
" agent_factory = StandardAgentFactory(\n",
|
||||
" task, model, prompt, rate_limiter=rate_limiter\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" client.run_on_dataset(\n",
|
||||
" dataset_name=dataset_name,\n",
|
||||
" llm_or_chain_factory=agent_factory,\n",
|
||||
" evaluation=eval_config,\n",
|
||||
" verbose=False,\n",
|
||||
" project_name=f\"{model_name}-{task.name}-{today}-{experiment_id}\",\n",
|
||||
" concurrency_level=5,\n",
|
||||
" project_metadata={\n",
|
||||
" \"model\": model_name,\n",
|
||||
" \"id\": experiment_uuid,\n",
|
||||
" \"task\": task.name,\n",
|
||||
" \"date\": today,\n",
|
||||
" \"langchain_benchmarks_version\": __version__,\n",
|
||||
" },\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4c0a6505-693d-46e5-9ed1-e33e0044b040",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Advanced Usage\n",
|
||||
"\n",
|
||||
"The following sections demonstrate slightly more \"advanced\" usage if you want to completely customize the agent runtime in a way that is compatible with our test runner.\n",
|
||||
"\n",
|
||||
"We'll also apply an adapter to the agent which will will capture its inputs and outputs (e.g, add information the agent's environment at the end of the run) so that it we can evaluate it.\n",
|
||||
"\n",
|
||||
"### Custom Agent Factory\n",
|
||||
"\n",
|
||||
"If you want even more configurability beyond what the `CustomRunnableAgentFactory` provides, you can create your owne `AgentFactory` using the following pattern.\n",
|
||||
"\n",
|
||||
"The `AgentExecutor` should accept `question` as an input and include the fields `output`, `intermediate_steps` and potentially `state` in its response -- for this we\n",
|
||||
"will wrap the agent executor in an adapter (`apply_agent_executor_adapter`) that will help match the expected schema."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "69351864-2e97-43df-81ae-5067cbf5e471",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import Optional\n",
|
||||
"\n",
|
||||
"from langchain.agents import AgentExecutor, create_tool_calling_agent\n",
|
||||
"from langchain_anthropic import ChatAnthropic\n",
|
||||
"from langchain_core.prompts import ChatPromptTemplate\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.schema import ExtractionTask\n",
|
||||
"from langchain_benchmarks.tool_usage.agents import apply_agent_executor_adapter\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class CustomAgentFactory:\n",
|
||||
" def __init__(\n",
|
||||
" self,\n",
|
||||
" task: ExtractionTask,\n",
|
||||
" *,\n",
|
||||
" # It can be useful to add a rate-limiter\n",
|
||||
" # which will limit ther number of requests per second\n",
|
||||
" # when running evaluation.\n",
|
||||
" rate_limiter: Optional[RateLimiter] = None,\n",
|
||||
" ) -> None:\n",
|
||||
" self.task = task\n",
|
||||
" self.rate_limiter = rate_limiter\n",
|
||||
"\n",
|
||||
" def __call__(self):\n",
|
||||
" # This factory creates a new environment for every agent run.\n",
|
||||
" # The reason is that the environment may be associated with an environment state (e.g., typewriter)\n",
|
||||
" # which is changed by the actions of the agent.\n",
|
||||
" # At the end of the run, the environment state will be read.\n",
|
||||
" env = task.create_environment() # Create a new environment for every agent run!\n",
|
||||
" tools = env.tools\n",
|
||||
" model = ChatAnthropic(model=\"claude-3-opus-20240229\", temperature=0)\n",
|
||||
" prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", self.task.instructions),\n",
|
||||
" (\n",
|
||||
" \"human\",\n",
|
||||
" \"{question}\",\n",
|
||||
" ), # Populated from task.instructions automatically\n",
|
||||
" (\"placeholder\", \"{agent_scratchpad}\"),\n",
|
||||
" ]\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # This is the standard tool calling agent implementation\n",
|
||||
" # Feel free to replace it with any other implementation you want!\n",
|
||||
" # https://python.langchain.com/docs/modules/agents/how_to/custom_agent/\n",
|
||||
" agent = create_tool_calling_agent(model, env.tools, prompt)\n",
|
||||
"\n",
|
||||
" if self.rate_limiter:\n",
|
||||
" agent = with_rate_limit(agent, self.rate_limiter)\n",
|
||||
"\n",
|
||||
" executor = AgentExecutor(\n",
|
||||
" agent=agent,\n",
|
||||
" tools=env.tools,\n",
|
||||
" handle_parsing_errors=True,\n",
|
||||
" return_intermediate_steps=True,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Apply the adapters so that inputs and outputs match dataset schema\n",
|
||||
" # state_reader automatically adds the state of the environment at the end of the run.\n",
|
||||
" return apply_agent_executor_adapter(executor, state_reader=env.read_state)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"id": "18a96a6f-812b-4b0e-83c5-d001bf50851e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>Tool Usage - Typewriter (26 tools) </td></tr>\n",
|
||||
"<tr><td>Type </td><td>ToolUsageTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d\" target=\"_blank\" rel=\"noopener\">128af05e-aa00-4e3b-a958-d166dd450581</a></td></tr>\n",
|
||||
"<tr><td>Description</td><td>Environment with 26 tools each tool represents a letter of the alphabet.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the model's ability the use tools\n",
|
||||
"for a simple repetition task.\n",
|
||||
"\n",
|
||||
"For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
|
||||
"\n",
|
||||
"The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\n",
|
||||
"\n",
|
||||
"This is a variation of the typer writer task, where 26 parameterless tools are\n",
|
||||
"given instead of a single tool that takes a letter as an argument. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"ToolUsageTask(name='Tool Usage - Typewriter (26 tools)', dataset_id='https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d', description=\"Environment with 26 tools each tool represents a letter of the alphabet.\\n\\nThe objective of this task is to evaluate the model's ability the use tools\\nfor a simple repetition task.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\\nThis is a variation of the typer writer task, where 26 parameterless tools are\\ngiven instead of a single tool that takes a letter as an argument.\\n\", create_environment=<function get_environment at 0x78972c6c3060>, instructions=\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\", eval_params={'output_evaluation': 'none'})"
|
||||
]
|
||||
},
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"id": "a7bd4af3-c0f1-4308-abbf-330d7497b3e3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"custom_agent_factory = CustomAgentFactory(task)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"id": "c5b69b7c-4294-47d1-85d7-47d718945898",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"agent = custom_agent_factory()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"id": "1ac24ef5-d3ca-41aa-b888-7ebcd8a92ff4",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'question': 'abc',\n",
|
||||
" 'output': [],\n",
|
||||
" 'intermediate_steps': [(ToolAgentAction(tool='a', tool_input={}, log='\\nInvoking: `a` with `{}`\\nresponded: [{\\'text\\': \\'<thinking>\\\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\\\n</thinking>\\', \\'type\\': \\'text\\'}, {\\'id\\': \\'toolu_016f6CZwwFmdz2h8KbdGRVjj\\', \\'input\\': {}, \\'name\\': \\'a\\', \\'type\\': \\'tool_use\\'}, {\\'id\\': \\'toolu_01JvfeTpU3hEuS7PknFk5a8S\\', \\'input\\': {}, \\'name\\': \\'b\\', \\'type\\': \\'tool_use\\'}, {\\'id\\': \\'toolu_01NbBCY5Fg62RsyAAUd4n2g1\\', \\'input\\': {}, \\'name\\': \\'c\\', \\'type\\': \\'tool_use\\'}]\\n\\n', message_log=[AIMessageChunk(content=[{'text': '<thinking>\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\n</thinking>', 'type': 'text'}, {'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj', 'input': {}, 'name': 'a', 'type': 'tool_use'}, {'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S', 'input': {}, 'name': 'b', 'type': 'tool_use'}, {'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1', 'input': {}, 'name': 'c', 'type': 'tool_use'}], id='run-42ea263e-e52a-4fc7-8aa3-71e16a9db42b', tool_calls=[{'name': 'a', 'args': {}, 'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj'}, {'name': 'b', 'args': {}, 'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S'}, {'name': 'c', 'args': {}, 'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1'}], tool_call_chunks=[{'name': 'a', 'args': '{}', 'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj', 'index': 0}, {'name': 'b', 'args': '{}', 'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S', 'index': 1}, {'name': 'c', 'args': '{}', 'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1', 'index': 2}])], tool_call_id='toolu_016f6CZwwFmdz2h8KbdGRVjj'),\n",
|
||||
" 'OK'),\n",
|
||||
" (ToolAgentAction(tool='b', tool_input={}, log='\\nInvoking: `b` with `{}`\\nresponded: [{\\'text\\': \\'<thinking>\\\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\\\n</thinking>\\', \\'type\\': \\'text\\'}, {\\'id\\': \\'toolu_016f6CZwwFmdz2h8KbdGRVjj\\', \\'input\\': {}, \\'name\\': \\'a\\', \\'type\\': \\'tool_use\\'}, {\\'id\\': \\'toolu_01JvfeTpU3hEuS7PknFk5a8S\\', \\'input\\': {}, \\'name\\': \\'b\\', \\'type\\': \\'tool_use\\'}, {\\'id\\': \\'toolu_01NbBCY5Fg62RsyAAUd4n2g1\\', \\'input\\': {}, \\'name\\': \\'c\\', \\'type\\': \\'tool_use\\'}]\\n\\n', message_log=[AIMessageChunk(content=[{'text': '<thinking>\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\n</thinking>', 'type': 'text'}, {'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj', 'input': {}, 'name': 'a', 'type': 'tool_use'}, {'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S', 'input': {}, 'name': 'b', 'type': 'tool_use'}, {'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1', 'input': {}, 'name': 'c', 'type': 'tool_use'}], id='run-42ea263e-e52a-4fc7-8aa3-71e16a9db42b', tool_calls=[{'name': 'a', 'args': {}, 'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj'}, {'name': 'b', 'args': {}, 'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S'}, {'name': 'c', 'args': {}, 'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1'}], tool_call_chunks=[{'name': 'a', 'args': '{}', 'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj', 'index': 0}, {'name': 'b', 'args': '{}', 'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S', 'index': 1}, {'name': 'c', 'args': '{}', 'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1', 'index': 2}])], tool_call_id='toolu_01JvfeTpU3hEuS7PknFk5a8S'),\n",
|
||||
" 'OK'),\n",
|
||||
" (ToolAgentAction(tool='c', tool_input={}, log='\\nInvoking: `c` with `{}`\\nresponded: [{\\'text\\': \\'<thinking>\\\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\\\n</thinking>\\', \\'type\\': \\'text\\'}, {\\'id\\': \\'toolu_016f6CZwwFmdz2h8KbdGRVjj\\', \\'input\\': {}, \\'name\\': \\'a\\', \\'type\\': \\'tool_use\\'}, {\\'id\\': \\'toolu_01JvfeTpU3hEuS7PknFk5a8S\\', \\'input\\': {}, \\'name\\': \\'b\\', \\'type\\': \\'tool_use\\'}, {\\'id\\': \\'toolu_01NbBCY5Fg62RsyAAUd4n2g1\\', \\'input\\': {}, \\'name\\': \\'c\\', \\'type\\': \\'tool_use\\'}]\\n\\n', message_log=[AIMessageChunk(content=[{'text': '<thinking>\\nTo repeat the string \"abc\", I need to call the a(), b(), and c() functions in that order. No parameters are required for these functions.\\n</thinking>', 'type': 'text'}, {'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj', 'input': {}, 'name': 'a', 'type': 'tool_use'}, {'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S', 'input': {}, 'name': 'b', 'type': 'tool_use'}, {'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1', 'input': {}, 'name': 'c', 'type': 'tool_use'}], id='run-42ea263e-e52a-4fc7-8aa3-71e16a9db42b', tool_calls=[{'name': 'a', 'args': {}, 'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj'}, {'name': 'b', 'args': {}, 'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S'}, {'name': 'c', 'args': {}, 'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1'}], tool_call_chunks=[{'name': 'a', 'args': '{}', 'id': 'toolu_016f6CZwwFmdz2h8KbdGRVjj', 'index': 0}, {'name': 'b', 'args': '{}', 'id': 'toolu_01JvfeTpU3hEuS7PknFk5a8S', 'index': 1}, {'name': 'c', 'args': '{}', 'id': 'toolu_01NbBCY5Fg62RsyAAUd4n2g1', 'index': 2}])], tool_call_id='toolu_01NbBCY5Fg62RsyAAUd4n2g1'),\n",
|
||||
" 'OK')],\n",
|
||||
" 'state': 'abc'}"
|
||||
]
|
||||
},
|
||||
"execution_count": 20,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"agent.invoke({\"question\": \"abc\"})"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,319 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Multiverse Math\n",
|
||||
"\n",
|
||||
"In this task, the agent is operating in an alternate universe which in which the basic mathematical operations like addition and multiplication are different.\n",
|
||||
"\n",
|
||||
"The agent must use tools that allow is to carry out calculations in this universe.\n",
|
||||
"\n",
|
||||
"This task can help verify that an agent is able to ignore its own knowledge of math and instead correctly use information returned by the tools.\n",
|
||||
"\n",
|
||||
"The modified mathematical operations yield different reuslts, but still retain some properties (e.g., the modified multiplication operation is still commutative).\n",
|
||||
"\n",
|
||||
"Please note that the modified operations are not guaranteed to even make sense in the real world since not all properties will be retained (e.g., distributive property).\n",
|
||||
"\n",
|
||||
"------------------"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8e3b729e-b851-4ab8-a3a9-be34b329b985",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"For this code to work, please configure LangSmith environment variables with your credentials.\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_API_KEY\"] = \"ls_..\" # Your LangSmith API key\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks import registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "1aef2b32-a5df-421f-8be3-a2ef27372ece",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>Multiverse Math </td></tr>\n",
|
||||
"<tr><td>Type </td><td>ToolUsageTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/47ed57bc-e852-4f84-a23e-cce4793864e9/d\" target=\"_blank\" rel=\"noopener\">47ed57bc-e852-4f84-a23e-cce4793864e9</a></td></tr>\n",
|
||||
"<tr><td>Description</td><td>An environment that contains a few basic math operations, but with altered results.\n",
|
||||
"\n",
|
||||
"For example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math.\n",
|
||||
"\n",
|
||||
"This task is associated with 20 test examples. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"ToolUsageTask(name='Multiverse Math', dataset_id='https://smith.langchain.com/public/47ed57bc-e852-4f84-a23e-cce4793864e9/d', description='An environment that contains a few basic math operations, but with altered results.\\n\\nFor example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\\n\\nThe objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math.\\n\\nThis task is associated with 20 test examples.\\n', create_environment=<function get_environment at 0x721642cba020>, instructions='You are requested to solve math questions in an alternate mathematical universe. The operations have been altered to yield different results than expected. Do not guess the answer or rely on your innate knowledge of math. Use the provided tools to answer the question. While associativity and commutativity apply, distributivity does not. Answer the question using the fewest possible tools. Only include the numeric response without any clarifications.', eval_params={'output_evaluation': 'qa_math_without_question'})"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task = registry[\"Multiverse Math\"]\n",
|
||||
"task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bc33a639-3caf-4314-8ea7-1c7c8b1d114d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Clone the dataset associated with this task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cede4edd-884d-4330-a186-5058b712394b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## The Environment\n",
|
||||
"\n",
|
||||
"Let's check the environment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "e2439d0c-ccb9-4f5b-a127-548725025a98",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[StructuredTool(name='multiply', description='multiply(a: float, b: float) -> float - Multiply two numbers; a * b.', args_schema=<class 'pydantic.v1.main.multiplySchema'>, func=<function multiply at 0x7216a3a78220>),\n",
|
||||
" StructuredTool(name='add', description='add(a: float, b: float) -> float - Add two numbers; a + b.', args_schema=<class 'pydantic.v1.main.addSchema'>, func=<function add at 0x721642cb9b20>),\n",
|
||||
" StructuredTool(name='divide', description='divide(a: float, b: float) -> float - Divide two numbers; a / b.', args_schema=<class 'pydantic.v1.main.divideSchema'>, func=<function divide at 0x72167803be20>),\n",
|
||||
" StructuredTool(name='subtract', description='subtract(a: float, b: float) -> float - Subtract two numbers; a - b.', args_schema=<class 'pydantic.v1.main.subtractSchema'>, func=<function subtract at 0x721642cb9d00>),\n",
|
||||
" StructuredTool(name='power', description='power(a: float, b: float) -> float - Raise a number to a power; a ** b.', args_schema=<class 'pydantic.v1.main.powerSchema'>, func=<function power at 0x721642cb9da0>)]"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env = task.create_environment()\n",
|
||||
"env.tools[:5]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1941e187-55ee-4d38-b529-4744ea2474b0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Multiplying 2 x 4 = 8.8!!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "f5a100bd-6e19-498f-8a36-393b5c19bcb9",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"8.8"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.tools[0].invoke({\"a\": 2, \"b\": 4})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bc60ef11-6300-4a83-989e-ec5b7f196796",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The task instructions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "31afb08b-17b8-4866-86c1-ee24e804415c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'You are requested to solve math questions in an alternate mathematical universe. The operations have been altered to yield different results than expected. Do not guess the answer or rely on your innate knowledge of math. Use the provided tools to answer the question. While associativity and commutativity apply, distributivity does not. Answer the question using the fewest possible tools. Only include the numeric response without any clarifications.'"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task.instructions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "92d65770-6a4f-4029-beba-5fa9aeb18809",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Agent Factory\n",
|
||||
"\n",
|
||||
"For evaluation, we need an agent factory that will create a new instance of an agent executor for every evaluation run.\n",
|
||||
"\n",
|
||||
"We'll use an `OpenAIAgentFactory` provided with LangChain Benchmarks -- look at the `intro` section to see how to define your own."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "6142cf4e-862c-47a3-aa75-81d7d3231308",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_core.prompts import ChatPromptTemplate\n",
|
||||
"from langchain_openai.chat_models import ChatOpenAI\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.tool_usage.agents import StandardAgentFactory\n",
|
||||
"\n",
|
||||
"model = ChatOpenAI(temperature=0)\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", \"{instructions}\"), # Populated from task.instructions automatically\n",
|
||||
" (\"human\", \"{question}\"), # Populated from the test data\n",
|
||||
" (\n",
|
||||
" \"placeholder\",\n",
|
||||
" \"{agent_scratchpad}\",\n",
|
||||
" ), # Work where the agent can do its work (e.g., call multiple tools)\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"agent_factory = StandardAgentFactory(task, model, prompt)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "11e4fff5-e184-45e1-a472-c0a9f70e897a",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
|
||||
"\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `add` with `{'a': 2, 'b': 5}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[33;1m\u001b[1;3m8.2\u001b[0m\u001b[32;1m\u001b[1;3mThe result of 2 + 5 in this alternate mathematical universe is 8.2.\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'question': 'how much is 2+5',\n",
|
||||
" 'output': 'The result of 2 + 5 in this alternate mathematical universe is 8.2.',\n",
|
||||
" 'intermediate_steps': [(ToolAgentAction(tool='add', tool_input={'a': 2, 'b': 5}, log=\"\\nInvoking: `add` with `{'a': 2, 'b': 5}`\\n\\n\\n\", message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_MZMnEZrae7AuXYtWzH0l9xKL', 'function': {'arguments': '{\"a\":2,\"b\":5}', 'name': 'add'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-b7548303-194d-40ee-85bf-3d43cac39526', tool_calls=[{'name': 'add', 'args': {'a': 2, 'b': 5}, 'id': 'call_MZMnEZrae7AuXYtWzH0l9xKL'}], tool_call_chunks=[{'name': 'add', 'args': '{\"a\":2,\"b\":5}', 'id': 'call_MZMnEZrae7AuXYtWzH0l9xKL', 'index': 0}])], tool_call_id='call_MZMnEZrae7AuXYtWzH0l9xKL'),\n",
|
||||
" 8.2)]}"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain import globals\n",
|
||||
"\n",
|
||||
"globals.set_verbose(True)\n",
|
||||
"\n",
|
||||
"agent = agent_factory()\n",
|
||||
"agent.invoke({\"question\": \"how much is 2+5\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b29a915c-1041-4108-a234-a877b6f59de4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Benchmarking\n",
|
||||
"\n",
|
||||
"See `introduction` and `benchmark all` for information on how to run benchmarks. This notebook is just to here to explain and explore the task."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,397 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Evaluating OSS Models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8e3b729e-b851-4ab8-a3a9-be34b329b985",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"For this code to work, please configure LangSmith environment variables with your credentials.\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_API_KEY\"] = \"ls_..\" # Your LangSmith API key\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"id": "666a8246-b1a9-47ce-b159-d950692fc06b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import getpass\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"keys = [\"LANGCHAIN_API_KEY\", \"FIREWORKS_API_KEY\"]\n",
|
||||
"for key in keys:\n",
|
||||
" if not os.environ.get(key):\n",
|
||||
" os.environ[key] = getpass(f\"Set {key}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "92d65770-6a4f-4029-beba-5fa9aeb18809",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Agent Factory\n",
|
||||
"\n",
|
||||
"For evaluation, we need an agent factory that will create a new instance of an agent executor for every evaluation run.\n",
|
||||
"\n",
|
||||
"We'll use an custom AgentFactory provided with LangChain Benchmarks -- look at the `intro` section to see how to define your own.\n",
|
||||
"\n",
|
||||
"We will use the Fireworks API for this."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "a35cbf20-7632-4116-9c6c-cee6e4a98068",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"from functools import partial\n",
|
||||
"from typing import Sequence, Tuple\n",
|
||||
"\n",
|
||||
"from langchain.agents import AgentExecutor\n",
|
||||
"from langchain.agents.structured_chat.output_parser import (\n",
|
||||
" AgentAction,\n",
|
||||
" AgentFinish,\n",
|
||||
")\n",
|
||||
"from langchain.output_parsers.json import parse_json_markdown\n",
|
||||
"from langchain.prompts import ChatPromptTemplate\n",
|
||||
"from langchain.tools import tool\n",
|
||||
"from langchain_core.runnables import RunnableLambda\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks import clone_public_dataset, registry\n",
|
||||
"from langchain_benchmarks.schema import BaseTask, RegisteredModel\n",
|
||||
"from langchain_benchmarks.tool_usage import apply_agent_executor_adapter\n",
|
||||
"from langchain_benchmarks.tool_usage.agents import apply_agent_executor_adapter\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"@tool\n",
|
||||
"def final_answer(answer: str) -> str:\n",
|
||||
" \"\"\"The final answer to the question.\"\"\"\n",
|
||||
" return answer\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def extract_first_json_object(text):\n",
|
||||
" # A hacky FSM to get the first JSON object across newlines\n",
|
||||
" OUTSIDE, INSIDE, IN_STRING = range(3)\n",
|
||||
"\n",
|
||||
" state = OUTSIDE\n",
|
||||
" nested_level = 0\n",
|
||||
" start_index = None\n",
|
||||
"\n",
|
||||
" def is_escaped(index):\n",
|
||||
" escape = False\n",
|
||||
" while index > 0 and text[index - 1] == \"\\\\\":\n",
|
||||
" escape = not escape\n",
|
||||
" index -= 1\n",
|
||||
" return escape\n",
|
||||
"\n",
|
||||
" for i, char in enumerate(text):\n",
|
||||
" if state == OUTSIDE:\n",
|
||||
" if char == \"{\":\n",
|
||||
" state = INSIDE\n",
|
||||
" nested_level = 1\n",
|
||||
" start_index = i\n",
|
||||
"\n",
|
||||
" elif state == INSIDE:\n",
|
||||
" if char == '\"' and not is_escaped(i):\n",
|
||||
" state = IN_STRING\n",
|
||||
" elif char == \"{\":\n",
|
||||
" nested_level += 1\n",
|
||||
" elif char == \"}\":\n",
|
||||
" nested_level -= 1\n",
|
||||
" if nested_level == 0:\n",
|
||||
" return text[start_index : i + 1]\n",
|
||||
"\n",
|
||||
" elif state == IN_STRING:\n",
|
||||
" if char == '\"' and not is_escaped(i):\n",
|
||||
" state = INSIDE\n",
|
||||
"\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def parse(message, prefix: str = \"\") -> dict:\n",
|
||||
" content = prefix + message.content.replace(\"\\_\", \"_\")\n",
|
||||
" content = extract_first_json_object(content)\n",
|
||||
" try:\n",
|
||||
" response = json.loads(content)\n",
|
||||
" except json.JSONDecodeError:\n",
|
||||
" response = parse_json_markdown(content)\n",
|
||||
" if response[\"action\"] == \"final_answer\":\n",
|
||||
" return AgentFinish({\"output\": response[\"action_input\"]}, content)\n",
|
||||
" else:\n",
|
||||
" return AgentAction(\n",
|
||||
" response[\"action\"],\n",
|
||||
" response.get(\"action_input\", {}),\n",
|
||||
" content,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def format_intermediate_steps(\n",
|
||||
" intermediate_steps: Sequence[Tuple[AgentAction, str]],\n",
|
||||
") -> str:\n",
|
||||
" if not intermediate_steps:\n",
|
||||
" return \"\"\n",
|
||||
"\n",
|
||||
" # response_tmpl = \"{action}\\n{{\\\"response\\\": \\\"{observation}\\\"}}\"\n",
|
||||
" response_tmpl = \"{action}\\n# Returned {observation}\"\n",
|
||||
" serialized = \"\\n\".join(\n",
|
||||
" [\n",
|
||||
" # f\"{agent_action.log.strip()}\\n{{\\\"response\\\": \\\"{observation}\\\"}}\"\n",
|
||||
" response_tmpl.format(\n",
|
||||
" action=agent_action.log.strip(), observation=observation\n",
|
||||
" )\n",
|
||||
" for agent_action, observation in intermediate_steps\n",
|
||||
" ]\n",
|
||||
" )\n",
|
||||
" return f\"\"\"\n",
|
||||
"```log.txt\n",
|
||||
"{serialized}\n",
|
||||
"```\n",
|
||||
"Consider previous steps above. What's your next step?\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def format_scratchpad(x):\n",
|
||||
" intermediate_steps = x[\"intermediate_steps\"]\n",
|
||||
" return format_intermediate_steps(intermediate_steps)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class AgentFactory:\n",
|
||||
" def __init__(\n",
|
||||
" self, task: BaseTask, model: RegisteredModel, num_retries: int = 5\n",
|
||||
" ) -> None:\n",
|
||||
" self.task = task\n",
|
||||
" self.model = model\n",
|
||||
" self.num_retries = num_retries\n",
|
||||
"\n",
|
||||
" def create_this_ugly_thing(self, env):\n",
|
||||
" tools = env.tools\n",
|
||||
"\n",
|
||||
" # schemas = []\n",
|
||||
" # for tool in tools + [final_answer]:\n",
|
||||
" # function_def = convert_to_openai_function(tool.args_schema)\n",
|
||||
" # function_def[\"name\"] = tool.name\n",
|
||||
" # schemas.append(function_def)\n",
|
||||
" # tools_str = \"\\n\".join([json.dumps(sc) for sc in schemas])\n",
|
||||
" tools_str = \"\\n\".join([tool.description for tool in tools + [final_answer]])\n",
|
||||
" messages = [\n",
|
||||
" (\n",
|
||||
" \"system\",\n",
|
||||
" f\"Task Instructions: {self.task.instructions}\\n\\n\"\n",
|
||||
" \"The following tools are exposed via an API:\\n\"\n",
|
||||
" \"{tools}\\n\\n\"\n",
|
||||
" \"Respond with one JSONL line to make your next action and call the API of a single tool.\"\n",
|
||||
" \"\"\" Format invocations like this:\n",
|
||||
"{{\"action\": \"tool name\",\"action_input\": {{TOOL BODY}}}}\n",
|
||||
"\\n\\nUse the final_answer tool only once you know the correct answer and have called the tools required for the task.\"\"\",\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" \"user\",\n",
|
||||
" \"{input}{agent_scratchpad}\\n\\nNote: Remember to respond in 1 JSONL line.\",\n",
|
||||
" ),\n",
|
||||
" ]\n",
|
||||
" parse_fn = parse\n",
|
||||
" if self.model.type == \"llm\":\n",
|
||||
" messages += [(\"assistant\", \"{{\")]\n",
|
||||
" # Fill it back in\n",
|
||||
" parse_fn = partial(parse_fn, prefix=\"{\")\n",
|
||||
" prompt = ChatPromptTemplate.from_messages(messages)\n",
|
||||
" prompt = prompt.partial(tools=tools_str)\n",
|
||||
"\n",
|
||||
" llm = self.model.get_model(model_params={\"temperature\": 0}).bind(stop=[\"\\n\\n\"])\n",
|
||||
" if self.num_retries:\n",
|
||||
" llm = llm.with_retry(stop_after_attempt=self.num_retries)\n",
|
||||
"\n",
|
||||
" @RunnableLambda\n",
|
||||
" def empty_fallback(x):\n",
|
||||
" \"\"\"Return an empty response to avoid misleading metrics.\"\"\"\n",
|
||||
" return {\n",
|
||||
" \"intermediate_steps\": [],\n",
|
||||
" \"state\": None,\n",
|
||||
" \"output\": \"ERROR\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" agent = (\n",
|
||||
" {\n",
|
||||
" \"input\": lambda x: x[\"input\"],\n",
|
||||
" \"agent_scratchpad\": format_scratchpad,\n",
|
||||
" }\n",
|
||||
" | prompt\n",
|
||||
" | llm\n",
|
||||
" | parse_fn\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" return AgentExecutor(\n",
|
||||
" agent=agent, tools=tools, return_intermediate_steps=True\n",
|
||||
" ).with_fallbacks([empty_fallback])\n",
|
||||
"\n",
|
||||
" def __call__(self):\n",
|
||||
" # This factory creates a new environment for every agent run.\n",
|
||||
" # The reason is that the environment may be associated with an environment state (e.g., typewriter)\n",
|
||||
" # which is changed by the actions of the agent.\n",
|
||||
" # At the end of the run, the environment state will be read.\n",
|
||||
" env = self.task.create_environment()\n",
|
||||
" executor = self.create_this_ugly_thing(env)\n",
|
||||
" # Apply the adapters so that inputs and outputs match dataset schema\n",
|
||||
" # state_reader automatically adds the state of the environment at the end of the run.\n",
|
||||
" return apply_agent_executor_adapter(executor, state_reader=env.read_state)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3821e4b0-8e67-418a-840c-470fcde42df0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Eval\n",
|
||||
"\n",
|
||||
"Let's evaluate an agent now"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "fd6cead0-3c37-4a73-8795-7819220797ee",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks.model_registration import model_registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"id": "fb32763c-79ab-426a-8fc6-bf8ebb0dd432",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"[-------> ] 3/20View the evaluation results for project 'mixtral-8x7b-fw-chat-ece3-Tool Usage - Typewriter (1 tool)' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/82ca6840-cf23-4bb0-a9be-55237ebbe9d3/compare?selectedSessions=2b92de52-2830-40cb-a396-4c08e0bf1c9b\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Tool Usage - Typewriter (1 tool) at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/82ca6840-cf23-4bb0-a9be-55237ebbe9d3\n",
|
||||
"[------------------------------------------------->] 20/20\n",
|
||||
"View the evaluation results for project 'mixtral-8x7b-ece3-Tool Usage - Typewriter (1 tool)' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/82ca6840-cf23-4bb0-a9be-55237ebbe9d3/compare?selectedSessions=ff797831-aee8-43db-a814-7727f9240006\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Tool Usage - Typewriter (1 tool) at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/82ca6840-cf23-4bb0-a9be-55237ebbe9d3\n",
|
||||
"[------------------------------------------------->] 20/20\n",
|
||||
"View the evaluation results for project 'mixtral-8x7b-fw-chat-ece3-Tool Usage - Typewriter (26 tools)' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/2f462c7a-f9b9-46e7-b96b-7469e965f478/compare?selectedSessions=1adbc135-93d9-46b2-a33a-e5470eded263\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Tool Usage - Typewriter (26 tools) at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/2f462c7a-f9b9-46e7-b96b-7469e965f478\n",
|
||||
"[------------------------------------------------->] 20/20\n",
|
||||
"View the evaluation results for project 'mixtral-8x7b-ece3-Tool Usage - Typewriter (26 tools)' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/2f462c7a-f9b9-46e7-b96b-7469e965f478/compare?selectedSessions=a8548cef-4afd-4f7e-9d21-7bd2fb3f9033\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Tool Usage - Typewriter (26 tools) at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/2f462c7a-f9b9-46e7-b96b-7469e965f478\n",
|
||||
"[------------------------------------------------->] 20/20\n",
|
||||
"View the evaluation results for project 'mixtral-8x7b-fw-chat-ece3-Tool Usage - Relational Data' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/df6be6c9-05b3-445e-8836-ebb4aba63826/compare?selectedSessions=685df1fb-605d-40e3-b645-ae132a0a6229\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Tool Usage - Relational Data at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/df6be6c9-05b3-445e-8836-ebb4aba63826\n",
|
||||
"[------------------------------------------------->] 21/21\n",
|
||||
"View the evaluation results for project 'mixtral-8x7b-ece3-Tool Usage - Relational Data' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/df6be6c9-05b3-445e-8836-ebb4aba63826/compare?selectedSessions=bb4d1ee4-bbc8-4969-a4f0-2b0732444785\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Tool Usage - Relational Data at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/df6be6c9-05b3-445e-8836-ebb4aba63826\n",
|
||||
"[------------------------------------------------->] 21/21\n",
|
||||
"View the evaluation results for project 'mixtral-8x7b-fw-chat-ece3-Multiverse Math' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/108bdc68-1808-4b60-92ef-fbd9bd7e1ad0/compare?selectedSessions=ac7ec5aa-108d-4c5b-9c30-8e954fa132aa\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Multiverse Math at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/108bdc68-1808-4b60-92ef-fbd9bd7e1ad0\n",
|
||||
"[------------------------------------------------->] 10/10\n",
|
||||
"View the evaluation results for project 'mixtral-8x7b-ece3-Multiverse Math' at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/108bdc68-1808-4b60-92ef-fbd9bd7e1ad0/compare?selectedSessions=9d8573ee-847f-400a-8894-2e77c62e76ab\n",
|
||||
"\n",
|
||||
"View all tests for Dataset Multiverse Math at:\n",
|
||||
"https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/108bdc68-1808-4b60-92ef-fbd9bd7e1ad0\n",
|
||||
"[------------------------------------------------->] 10/10"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"from langsmith.client import Client\n",
|
||||
"\n",
|
||||
"experiment_uuid = uuid.uuid4().hex[:4]\n",
|
||||
"\n",
|
||||
"client = Client()\n",
|
||||
"\n",
|
||||
"task_names = [task.name for task in registry.filter(Type=\"ToolUsageTask\")]\n",
|
||||
"models = [\"mixtral-8x7b-fw-chat\", \"mixtral-8x7b\"]\n",
|
||||
"\n",
|
||||
"for task_name in task_names:\n",
|
||||
" for model_name in models:\n",
|
||||
" print()\n",
|
||||
" model = model_registry[model_name]\n",
|
||||
" task = registry[task_name]\n",
|
||||
" clone_public_dataset(task.dataset_id, dataset_name=task.name)\n",
|
||||
" eval_config = task.get_eval_config()\n",
|
||||
" test_run = client.run_on_dataset(\n",
|
||||
" dataset_name=task.name,\n",
|
||||
" llm_or_chain_factory=AgentFactory(task, model),\n",
|
||||
" evaluation=eval_config,\n",
|
||||
" project_name=f\"{model.name}-{experiment_uuid}-{task.name}\",\n",
|
||||
" tags=[model.name],\n",
|
||||
" project_metadata={\"id\": experiment_uuid, **model.params},\n",
|
||||
" verbose=True,\n",
|
||||
" )"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,305 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Relational Data \n",
|
||||
"\n",
|
||||
"In this task, an agent is given access to a set of tools that can be used to make queries across 3 relational tables.\n",
|
||||
"\n",
|
||||
"The tables contain information about users, locations and foods. The agent must answer questions about the data using the provided tools.\n",
|
||||
"\n",
|
||||
"The underlying data looks like this (showing first 2 records)\n",
|
||||
"\n",
|
||||
"User data:\n",
|
||||
"\n",
|
||||
"| id | name | email | location | favorite_color | favorite_foods |\n",
|
||||
"| ---- | -------- | ----------------- | -------- | --------------- | --------------- |\n",
|
||||
"| 1 | Alice | alice@gmail.com | 1 | red | [1, 2, 3] |\n",
|
||||
"| 21 | Bob | bob@hotmail.com | 2 | orange | [4, 5, 6] |\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Location Data:\n",
|
||||
"\n",
|
||||
"| id | city | current_time | current_weather |\n",
|
||||
"| ---- | ------------ | --------------------- | --------------------------------------- |\n",
|
||||
"| 1 | New York | 2023-11-14 10:30 AM | Partly Cloudy, Temperature: 68°F |\n",
|
||||
"| 2 | Los Angeles | 2023-11-14 7:45 AM | Sunny, Temperature: 75°F |\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Food data:\n",
|
||||
"\n",
|
||||
"| id | name | calories | allergic_ingredients |\n",
|
||||
"| ---- | ---------- | -------- | ------------------------ |\n",
|
||||
"| 1 | Pizza | 285 | [\"Gluten\", \"Dairy\"] |\n",
|
||||
"| 2 | Chocolate | 50 | [\"Milk\", \"Soy\"] |\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"The tools allow to look up information based on ids (e.g., `get_user_email` takes a user id and returns the email),\n",
|
||||
"and to search (e.g., `find_foods_by_name` takes a food name and returns a list of results).\n",
|
||||
"\n",
|
||||
"----------"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks import registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "03488ab1-31ed-41c2-8da2-46b02599b181",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"For this code to work, please configure LangSmith environment variables with your credentials."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "60f22779-a948-4833-8e8c-ace9ef17f56f",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"task = registry[\"Tool Usage - Relational Data\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "110bdafa-bdab-4194-90c9-46416d14b2f9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## The Environment\n",
|
||||
"\n",
|
||||
"Let's check the environment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "27b6b0fd-639d-43a7-a730-9acdc5b2f102",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[StructuredTool(name='get_user_name', description=\"get_user_name(user_id: int) -> str - Get the name of the user with the given user ID.\\n\\n Args:\\n user_id: The user's ID.\\n\\n Returns:\\n The user's name.\", args_schema=<class 'pydantic.v1.main.get_user_nameSchema'>, handle_tool_error=True, func=<function get_available_functions.<locals>.get_user_name at 0x78f30602fec0>),\n",
|
||||
" StructuredTool(name='list_user_ids', description='list_user_ids() -> List[str] - List all the user IDs.', args_schema=<class 'pydantic.v1.main.list_user_idsSchema'>, handle_tool_error=True, func=<function get_available_functions.<locals>.list_user_ids at 0x78f30602fe20>),\n",
|
||||
" StructuredTool(name='find_users_by_name', description='find_users_by_name(name: str) -> List[langchain_benchmarks.tool_usage.tasks.relational_data.SearchHit] - Find users with the given name.\\n\\n Args:\\n name: The name to search for.\\n\\n Returns:\\n The list of matching users.', args_schema=<class 'pydantic.v1.main.find_users_by_nameSchema'>, handle_tool_error=True, func=<function get_available_functions.<locals>.find_users_by_name at 0x78f306058040>),\n",
|
||||
" StructuredTool(name='find_locations_by_name', description='find_locations_by_name(city: str) -> List[langchain_benchmarks.tool_usage.tasks.relational_data.SearchHit] - Find locations with the given city name.', args_schema=<class 'pydantic.v1.main.find_locations_by_nameSchema'>, handle_tool_error=True, func=<function get_available_functions.<locals>.find_locations_by_name at 0x78f3060580e0>),\n",
|
||||
" StructuredTool(name='find_foods_by_name', description='find_foods_by_name(food: str) -> List[langchain_benchmarks.tool_usage.tasks.relational_data.SearchHit] - Find foods with the given name.', args_schema=<class 'pydantic.v1.main.find_foods_by_nameSchema'>, handle_tool_error=True, func=<function get_available_functions.<locals>.find_foods_by_name at 0x78f306058180>)]"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env = task.create_environment()\n",
|
||||
"env.tools[:5]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "7f1c1242-449c-4536-863d-b62bf6d2dff1",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'Bob'"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.tools[0].invoke({\"user_id\": 21})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "854e139b-a120-4012-bdf4-6394e0b1c42d",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[{'id': 2, 'city': 'Los Angeles'},\n",
|
||||
" {'id': 1, 'city': 'New York'},\n",
|
||||
" {'id': 3, 'city': 'Chicago'},\n",
|
||||
" {'id': 4, 'city': 'Houston'},\n",
|
||||
" {'id': 5, 'city': 'Miami'}]"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.tools[3].invoke({\"city\": \"LA\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b462f7b8-fd42-4613-ab5f-5f3cbbc37d28",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Explore the task\n",
|
||||
"\n",
|
||||
"For evaluation, we need an agent factory that will create a new instance of an agent executor for every evaluation run.\n",
|
||||
"\n",
|
||||
"We'll use the `StandardAgentFactory` -- look at the `intro` for more information about what it does and/or how to create a custom one."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "81c0e4a1-f56e-4117-8804-4161c642b068",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_core.prompts import ChatPromptTemplate\n",
|
||||
"from langchain_openai.chat_models import ChatOpenAI\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.tool_usage.agents import StandardAgentFactory\n",
|
||||
"\n",
|
||||
"model = ChatOpenAI(temperature=0)\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", \"{instructions}\"), # Populated from task.instructions automatically\n",
|
||||
" (\"human\", \"{question}\"), # Populated from the test data\n",
|
||||
" (\n",
|
||||
" \"placeholder\",\n",
|
||||
" \"{agent_scratchpad}\",\n",
|
||||
" ), # Work where the agent can do its work (e.g., call multiple tools)\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"agent_factory = StandardAgentFactory(task, model, prompt)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "382ff2f6-8099-415e-a58c-e659345f52fc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
|
||||
"\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `find_locations_by_name` with `{'city': 'LA'}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3m[{'id': 2, 'city': 'Los Angeles'}, {'id': 1, 'city': 'New York'}, {'id': 3, 'city': 'Chicago'}, {'id': 4, 'city': 'Houston'}, {'id': 5, 'city': 'Miami'}]\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `get_current_weather_for_location` with `{'location_id': 2}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mSunny, Temperature: 75°F\u001b[0m\u001b[32;1m\u001b[1;3mThe weather in Los Angeles is sunny with a temperature of 75°F.\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'question': 'what is the weather in LA',\n",
|
||||
" 'output': 'The weather in Los Angeles is sunny with a temperature of 75°F.',\n",
|
||||
" 'intermediate_steps': [(ToolAgentAction(tool='find_locations_by_name', tool_input={'city': 'LA'}, log=\"\\nInvoking: `find_locations_by_name` with `{'city': 'LA'}`\\n\\n\\n\", message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_hJrCZgP4eDgaj6s4RtCKXTOo', 'function': {'arguments': '{\"city\":\"LA\"}', 'name': 'find_locations_by_name'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-23ccffb0-3b17-46a4-b42e-5eaa3220b211', tool_calls=[{'name': 'find_locations_by_name', 'args': {'city': 'LA'}, 'id': 'call_hJrCZgP4eDgaj6s4RtCKXTOo'}], tool_call_chunks=[{'name': 'find_locations_by_name', 'args': '{\"city\":\"LA\"}', 'id': 'call_hJrCZgP4eDgaj6s4RtCKXTOo', 'index': 0}])], tool_call_id='call_hJrCZgP4eDgaj6s4RtCKXTOo'),\n",
|
||||
" [{'id': 2, 'city': 'Los Angeles'},\n",
|
||||
" {'id': 1, 'city': 'New York'},\n",
|
||||
" {'id': 3, 'city': 'Chicago'},\n",
|
||||
" {'id': 4, 'city': 'Houston'},\n",
|
||||
" {'id': 5, 'city': 'Miami'}]),\n",
|
||||
" (ToolAgentAction(tool='get_current_weather_for_location', tool_input={'location_id': 2}, log=\"\\nInvoking: `get_current_weather_for_location` with `{'location_id': 2}`\\n\\n\\n\", message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_lopYjo00MF9mZtnHtiisTqyp', 'function': {'arguments': '{\"location_id\":2}', 'name': 'get_current_weather_for_location'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-9bba5827-d98b-464d-8028-25eb4a05d227', tool_calls=[{'name': 'get_current_weather_for_location', 'args': {'location_id': 2}, 'id': 'call_lopYjo00MF9mZtnHtiisTqyp'}], tool_call_chunks=[{'name': 'get_current_weather_for_location', 'args': '{\"location_id\":2}', 'id': 'call_lopYjo00MF9mZtnHtiisTqyp', 'index': 0}])], tool_call_id='call_lopYjo00MF9mZtnHtiisTqyp'),\n",
|
||||
" 'Sunny, Temperature: 75°F')]}"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain import globals\n",
|
||||
"\n",
|
||||
"globals.set_verbose(True)\n",
|
||||
"\n",
|
||||
"agent = agent_factory()\n",
|
||||
"agent.invoke({\"question\": \"what is the weather in LA\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "142ac640-3ce0-4f38-89cd-8d24d65997e4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Benchmarking\n",
|
||||
"\n",
|
||||
"See `introduction` and `benchmark all` for information on how to run benchmarks. This notebook is just to here to explain and explore the task."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e49455cc-13c5-4ea6-bb4b-e61c39ea0267",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,347 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Typewriter: Single Tool\n",
|
||||
"\n",
|
||||
"In this task, an agent is given access to a single tool called \"type_letter\".\n",
|
||||
"This tool takes one argument called \"letter\" which is expected to be a character.\n",
|
||||
"\n",
|
||||
"The agent must repeat the input string from the user, printing one\n",
|
||||
"character a time on a piece of virtual paper.\n",
|
||||
"\n",
|
||||
"The agent is evaluated based on its ability to print the correct string using\n",
|
||||
"the \"type_letter\" tool.\n",
|
||||
"\n",
|
||||
"--------"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks import registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "1aef2b32-a5df-421f-8be3-a2ef27372ece",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>Tool Usage - Typewriter (1 tool) </td></tr>\n",
|
||||
"<tr><td>Type </td><td>ToolUsageTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/59577193-8938-4ccf-92a7-e8a96bcf4f86/d\" target=\"_blank\" rel=\"noopener\">59577193-8938-4ccf-92a7-e8a96bcf4f86</a></td></tr>\n",
|
||||
"<tr><td>Description</td><td>Environment with a single tool that accepts a single letter as input, and prints it on a piece of virtual paper.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the ability of the model to use the provided tools to repeat a given input string.\n",
|
||||
"\n",
|
||||
"For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
|
||||
"\n",
|
||||
"The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"ToolUsageTask(name='Tool Usage - Typewriter (1 tool)', dataset_id='https://smith.langchain.com/public/59577193-8938-4ccf-92a7-e8a96bcf4f86/d', description=\"Environment with a single tool that accepts a single letter as input, and prints it on a piece of virtual paper.\\n\\nThe objective of this task is to evaluate the ability of the model to use the provided tools to repeat a given input string.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\", create_environment=<function get_environment at 0x73a65909da80>, instructions=\"Repeat the given string using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must print the letters 'a', 'b', and 'c' one at a time and in that order. \", eval_params={'output_evaluation': 'none'})"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task = registry[\"Tool Usage - Typewriter (1 tool)\"]\n",
|
||||
"task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fc78a3e1-80da-4607-98c3-a99c2037e7ca",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## The Environment\n",
|
||||
"\n",
|
||||
"The environment consists of a single tool and a virtual paper.\n",
|
||||
"\n",
|
||||
"The tool accepts a single letter as an input and prints the leter on the virtual paper. If successful, the tool returns the output \"OK\".\n",
|
||||
"\n",
|
||||
"To determine what's written on the paper, one needs to read the environment state."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "64e538ae-5cf2-4cd5-a312-25ee6924e869",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"env = task.create_environment()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "5516a34b-1e9b-4f1e-9462-cfc4d5bc29f9",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[StructuredTool(name='type_letter', description='type_letter(letter: str) -> str - Print the given letter on the paper.', args_schema=<class 'pydantic.v1.main.type_letterSchema'>, func=<function create_typer.<locals>.type_letter at 0x73a65909ee80>)]"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "80501e1a-f1f6-4b38-8637-894503029d86",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tool = env.tools[0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "3f352e32-fdb6-4d9e-b1c4-3d78b4f50646",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'OK'"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tool.invoke({\"letter\": \"a\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "ec9c2e68-b55e-4087-bc1a-c38f4cfd401b",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'OK'"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tool.invoke({\"letter\": \"b\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "2cc5b174-25a4-4d5a-8535-56ecea62ea81",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'ab'"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.read_state()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cd13d120-1bf9-481c-9392-c15ebdd9d77f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Explore the task\n",
|
||||
"\n",
|
||||
"For evaluation, we need an agent factory that will create a new instance of an agent executor for every evaluation run.\n",
|
||||
"\n",
|
||||
"We'll use the `StandardAgentFactory` -- look at the `intro` for more information about what it does and/or how to create a custom one."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"id": "e2acab1e-78a7-4198-8e79-4529c95ce7e2",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_core.prompts import ChatPromptTemplate\n",
|
||||
"from langchain_openai.chat_models import ChatOpenAI\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.tool_usage.agents import StandardAgentFactory\n",
|
||||
"\n",
|
||||
"model = ChatOpenAI(temperature=0)\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", \"{instructions}\"), # Populated from task.instructions automatically\n",
|
||||
" (\"human\", \"{question}\"), # Populated from the test data\n",
|
||||
" (\n",
|
||||
" \"placeholder\",\n",
|
||||
" \"{agent_scratchpad}\",\n",
|
||||
" ), # Work where the agent can do its work (e.g., call multiple tools)\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"agent_factory = StandardAgentFactory(task, model, prompt)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"id": "ceaa8edf-292b-48a1-be94-e6bfea0e75b1",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
|
||||
"\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `type_letter` with `{'letter': 'a'}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `type_letter` with `{'letter': 'b'}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `type_letter` with `{'letter': 'c'}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3mabc\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'question': 'abc',\n",
|
||||
" 'output': 'abc',\n",
|
||||
" 'intermediate_steps': [(ToolAgentAction(tool='type_letter', tool_input={'letter': 'a'}, log=\"\\nInvoking: `type_letter` with `{'letter': 'a'}`\\n\\n\\n\", message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80', 'function': {'arguments': '{\"letter\": \"a\"}', 'name': 'type_letter'}, 'type': 'function'}, {'index': 1, 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq', 'function': {'arguments': '{\"letter\": \"b\"}', 'name': 'type_letter'}, 'type': 'function'}, {'index': 2, 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj', 'function': {'arguments': '{\"letter\": \"c\"}', 'name': 'type_letter'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-7d6be045-b9e2-4f24-991c-8e34ccd53b98', tool_calls=[{'name': 'type_letter', 'args': {'letter': 'a'}, 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80'}, {'name': 'type_letter', 'args': {'letter': 'b'}, 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq'}, {'name': 'type_letter', 'args': {'letter': 'c'}, 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj'}], tool_call_chunks=[{'name': 'type_letter', 'args': '{\"letter\": \"a\"}', 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80', 'index': 0}, {'name': 'type_letter', 'args': '{\"letter\": \"b\"}', 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq', 'index': 1}, {'name': 'type_letter', 'args': '{\"letter\": \"c\"}', 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj', 'index': 2}])], tool_call_id='call_f4exPQMfz4VWxFJw4LhyMc80'),\n",
|
||||
" 'OK'),\n",
|
||||
" (ToolAgentAction(tool='type_letter', tool_input={'letter': 'b'}, log=\"\\nInvoking: `type_letter` with `{'letter': 'b'}`\\n\\n\\n\", message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80', 'function': {'arguments': '{\"letter\": \"a\"}', 'name': 'type_letter'}, 'type': 'function'}, {'index': 1, 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq', 'function': {'arguments': '{\"letter\": \"b\"}', 'name': 'type_letter'}, 'type': 'function'}, {'index': 2, 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj', 'function': {'arguments': '{\"letter\": \"c\"}', 'name': 'type_letter'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-7d6be045-b9e2-4f24-991c-8e34ccd53b98', tool_calls=[{'name': 'type_letter', 'args': {'letter': 'a'}, 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80'}, {'name': 'type_letter', 'args': {'letter': 'b'}, 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq'}, {'name': 'type_letter', 'args': {'letter': 'c'}, 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj'}], tool_call_chunks=[{'name': 'type_letter', 'args': '{\"letter\": \"a\"}', 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80', 'index': 0}, {'name': 'type_letter', 'args': '{\"letter\": \"b\"}', 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq', 'index': 1}, {'name': 'type_letter', 'args': '{\"letter\": \"c\"}', 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj', 'index': 2}])], tool_call_id='call_DHOJfLJEKuOKdzBa8ZLRYJZq'),\n",
|
||||
" 'OK'),\n",
|
||||
" (ToolAgentAction(tool='type_letter', tool_input={'letter': 'c'}, log=\"\\nInvoking: `type_letter` with `{'letter': 'c'}`\\n\\n\\n\", message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80', 'function': {'arguments': '{\"letter\": \"a\"}', 'name': 'type_letter'}, 'type': 'function'}, {'index': 1, 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq', 'function': {'arguments': '{\"letter\": \"b\"}', 'name': 'type_letter'}, 'type': 'function'}, {'index': 2, 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj', 'function': {'arguments': '{\"letter\": \"c\"}', 'name': 'type_letter'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-7d6be045-b9e2-4f24-991c-8e34ccd53b98', tool_calls=[{'name': 'type_letter', 'args': {'letter': 'a'}, 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80'}, {'name': 'type_letter', 'args': {'letter': 'b'}, 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq'}, {'name': 'type_letter', 'args': {'letter': 'c'}, 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj'}], tool_call_chunks=[{'name': 'type_letter', 'args': '{\"letter\": \"a\"}', 'id': 'call_f4exPQMfz4VWxFJw4LhyMc80', 'index': 0}, {'name': 'type_letter', 'args': '{\"letter\": \"b\"}', 'id': 'call_DHOJfLJEKuOKdzBa8ZLRYJZq', 'index': 1}, {'name': 'type_letter', 'args': '{\"letter\": \"c\"}', 'id': 'call_EziJvB6jtUEg3CmXSsQ7OWBj', 'index': 2}])], tool_call_id='call_EziJvB6jtUEg3CmXSsQ7OWBj'),\n",
|
||||
" 'OK')],\n",
|
||||
" 'state': 'abc'}"
|
||||
]
|
||||
},
|
||||
"execution_count": 25,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain import globals\n",
|
||||
"\n",
|
||||
"globals.set_verbose(True)\n",
|
||||
"\n",
|
||||
"agent = agent_factory()\n",
|
||||
"agent.invoke({\"question\": \"abc\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4729e72c-3903-478a-b298-4a586af33912",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Benchmarking\n",
|
||||
"\n",
|
||||
"See `introduction` and `benchmark all` for information on how to run benchmarks. This notebook is just to here to explain and explore the task."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "87055296-62e1-4fa9-8868-5c213f4ea2e6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,348 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "15e64bc9-bdf7-4fe5-9839-5d160c425c61",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Typewriter: 26 Tools\n",
|
||||
"\n",
|
||||
"This is a variation of the typewriter task in which the agent has access to 26 parameterless tools.\n",
|
||||
"\n",
|
||||
"Each tool represents a letter of the alphabet (e.g., 'a', 'b', 'c').\n",
|
||||
"\n",
|
||||
"The agent can use each tool to \"print\" the corresponding letter on a piece of virtual paper.\n",
|
||||
"\n",
|
||||
"The objective for the agent is to \"print\" the user's input on the paper exactly.\n",
|
||||
"\n",
|
||||
"---------\n",
|
||||
"\n",
|
||||
"For this code to work, please configure LangSmith environment variables with your credentials.\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\" # Your api key.\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_benchmarks import registry"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "1aef2b32-a5df-421f-8be3-a2ef27372ece",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tbody>\n",
|
||||
"<tr><td>Name </td><td>Tool Usage - Typewriter (26 tools) </td></tr>\n",
|
||||
"<tr><td>Type </td><td>ToolUsageTask </td></tr>\n",
|
||||
"<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d\" target=\"_blank\" rel=\"noopener\">128af05e-aa00-4e3b-a958-d166dd450581</a></td></tr>\n",
|
||||
"<tr><td>Description</td><td>Environment with 26 tools each tool represents a letter of the alphabet.\n",
|
||||
"\n",
|
||||
"The objective of this task is to evaluate the model's ability the use tools\n",
|
||||
"for a simple repetition task.\n",
|
||||
"\n",
|
||||
"For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
|
||||
"\n",
|
||||
"The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\n",
|
||||
"\n",
|
||||
"This is a variation of the typer writer task, where 26 parameterless tools are\n",
|
||||
"given instead of a single tool that takes a letter as an argument. </td></tr>\n",
|
||||
"</tbody>\n",
|
||||
"</table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"ToolUsageTask(name='Tool Usage - Typewriter (26 tools)', dataset_id='https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d', description=\"Environment with 26 tools each tool represents a letter of the alphabet.\\n\\nThe objective of this task is to evaluate the model's ability the use tools\\nfor a simple repetition task.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\\nThis is a variation of the typer writer task, where 26 parameterless tools are\\ngiven instead of a single tool that takes a letter as an argument.\\n\", create_environment=<function get_environment at 0x75aa9dec2d40>, instructions=\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\", eval_params={'output_evaluation': 'none'})"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"task = registry[\"Tool Usage - Typewriter (26 tools)\"]\n",
|
||||
"task"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b462f7b8-fd42-4613-ab5f-5f3cbbc37d28",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's build an agent that we can use for evaluation."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6ce51f81-1b3a-4dda-a382-c2fed3013af1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## The Environment\n",
|
||||
"\n",
|
||||
"The environment consists of 26 tools and a virtual paper.\n",
|
||||
"\n",
|
||||
"Each tool is responsible for printing a letter on the paper that corresponds to it."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "61535a75-24f6-4727-9549-f76c263e9153",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"env = task.create_environment()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "f35a0a1d-5a1e-4de1-8d8c-c7c9a264a6c7",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[StructuredTool(name='a', description='a() -> str - Run to Type the letter \"a\".', args_schema=<class 'pydantic.v1.main.aSchema'>, func=<function _create_typing_func.<locals>.func at 0x75aa9defc180>),\n",
|
||||
" StructuredTool(name='b', description='b() -> str - Run to Type the letter \"b\".', args_schema=<class 'pydantic.v1.main.bSchema'>, func=<function _create_typing_func.<locals>.func at 0x75aa9defc220>),\n",
|
||||
" StructuredTool(name='c', description='c() -> str - Run to Type the letter \"c\".', args_schema=<class 'pydantic.v1.main.cSchema'>, func=<function _create_typing_func.<locals>.func at 0x75aa9defc2c0>),\n",
|
||||
" StructuredTool(name='d', description='d() -> str - Run to Type the letter \"d\".', args_schema=<class 'pydantic.v1.main.dSchema'>, func=<function _create_typing_func.<locals>.func at 0x75aa9defc360>),\n",
|
||||
" StructuredTool(name='e', description='e() -> str - Run to Type the letter \"e\".', args_schema=<class 'pydantic.v1.main.eSchema'>, func=<function _create_typing_func.<locals>.func at 0x75aa9defc400>)]"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.tools[:5]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "5bea0190-39ec-4f30-9a00-90136bc6bf0b",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'OK'"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.tools[0].invoke({})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "bf7444da-15a1-455a-b22e-639cbfff8432",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'OK'"
|
||||
]
|
||||
},
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.tools[3].invoke({})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "d12bd710-5c01-4539-a4b9-afbf03164923",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'ad'"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"env.read_state()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f1d62a13-3771-460f-b131-4443f669ca3d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Explore the task\n",
|
||||
"\n",
|
||||
"For evaluation, we need an agent factory that will create a new instance of an agent executor for every evaluation run.\n",
|
||||
"\n",
|
||||
"We'll use the `StandardAgentFactory` -- look at the `intro` for more information about what it does and/or how to create a custom one."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "6142cf4e-862c-47a3-aa75-81d7d3231308",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain_core.prompts import ChatPromptTemplate\n",
|
||||
"from langchain_openai.chat_models import ChatOpenAI\n",
|
||||
"\n",
|
||||
"from langchain_benchmarks.tool_usage.agents import StandardAgentFactory\n",
|
||||
"\n",
|
||||
"model = ChatOpenAI(temperature=0)\n",
|
||||
"prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [\n",
|
||||
" (\"system\", \"{instructions}\"), # Populated from task.instructions automatically\n",
|
||||
" (\"human\", \"{question}\"), # Populated from the test data\n",
|
||||
" (\n",
|
||||
" \"placeholder\",\n",
|
||||
" \"{agent_scratchpad}\",\n",
|
||||
" ), # Work where the agent can do its work (e.g., call multiple tools)\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"agent_factory = StandardAgentFactory(task, model, prompt)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "fb32763c-79ab-426a-8fc6-bf8ebb0dd432",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
|
||||
"\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `a` with `{}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `b` with `{}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[33;1m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `c` with `{}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[38;5;200m\u001b[1;3mOK\u001b[0m\u001b[32;1m\u001b[1;3mabcabcabc\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'question': 'abc',\n",
|
||||
" 'output': 'abcabcabc',\n",
|
||||
" 'intermediate_steps': [(ToolAgentAction(tool='a', tool_input={}, log='\\nInvoking: `a` with `{}`\\n\\n\\n', message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI', 'function': {'arguments': '{}', 'name': 'a'}, 'type': 'function'}, {'index': 1, 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW', 'function': {'arguments': '{}', 'name': 'b'}, 'type': 'function'}, {'index': 2, 'id': 'call_MRAOAgbi8vT445clqC8OybMR', 'function': {'arguments': '{}', 'name': 'c'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-9a1af767-29e4-4759-ab28-5b29236e8f22', tool_calls=[{'name': 'a', 'args': {}, 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI'}, {'name': 'b', 'args': {}, 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW'}, {'name': 'c', 'args': {}, 'id': 'call_MRAOAgbi8vT445clqC8OybMR'}], tool_call_chunks=[{'name': 'a', 'args': '{}', 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI', 'index': 0}, {'name': 'b', 'args': '{}', 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW', 'index': 1}, {'name': 'c', 'args': '{}', 'id': 'call_MRAOAgbi8vT445clqC8OybMR', 'index': 2}])], tool_call_id='call_OrpjShN5uNzw2Rsb1tWF6swI'),\n",
|
||||
" 'OK'),\n",
|
||||
" (ToolAgentAction(tool='b', tool_input={}, log='\\nInvoking: `b` with `{}`\\n\\n\\n', message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI', 'function': {'arguments': '{}', 'name': 'a'}, 'type': 'function'}, {'index': 1, 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW', 'function': {'arguments': '{}', 'name': 'b'}, 'type': 'function'}, {'index': 2, 'id': 'call_MRAOAgbi8vT445clqC8OybMR', 'function': {'arguments': '{}', 'name': 'c'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-9a1af767-29e4-4759-ab28-5b29236e8f22', tool_calls=[{'name': 'a', 'args': {}, 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI'}, {'name': 'b', 'args': {}, 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW'}, {'name': 'c', 'args': {}, 'id': 'call_MRAOAgbi8vT445clqC8OybMR'}], tool_call_chunks=[{'name': 'a', 'args': '{}', 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI', 'index': 0}, {'name': 'b', 'args': '{}', 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW', 'index': 1}, {'name': 'c', 'args': '{}', 'id': 'call_MRAOAgbi8vT445clqC8OybMR', 'index': 2}])], tool_call_id='call_2XO5RNgt9FjGvTXztgD0tKqW'),\n",
|
||||
" 'OK'),\n",
|
||||
" (ToolAgentAction(tool='c', tool_input={}, log='\\nInvoking: `c` with `{}`\\n\\n\\n', message_log=[AIMessageChunk(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI', 'function': {'arguments': '{}', 'name': 'a'}, 'type': 'function'}, {'index': 1, 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW', 'function': {'arguments': '{}', 'name': 'b'}, 'type': 'function'}, {'index': 2, 'id': 'call_MRAOAgbi8vT445clqC8OybMR', 'function': {'arguments': '{}', 'name': 'c'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-9a1af767-29e4-4759-ab28-5b29236e8f22', tool_calls=[{'name': 'a', 'args': {}, 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI'}, {'name': 'b', 'args': {}, 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW'}, {'name': 'c', 'args': {}, 'id': 'call_MRAOAgbi8vT445clqC8OybMR'}], tool_call_chunks=[{'name': 'a', 'args': '{}', 'id': 'call_OrpjShN5uNzw2Rsb1tWF6swI', 'index': 0}, {'name': 'b', 'args': '{}', 'id': 'call_2XO5RNgt9FjGvTXztgD0tKqW', 'index': 1}, {'name': 'c', 'args': '{}', 'id': 'call_MRAOAgbi8vT445clqC8OybMR', 'index': 2}])], tool_call_id='call_MRAOAgbi8vT445clqC8OybMR'),\n",
|
||||
" 'OK')],\n",
|
||||
" 'state': 'abc'}"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain import globals\n",
|
||||
"\n",
|
||||
"globals.set_verbose(True)\n",
|
||||
"\n",
|
||||
"agent = agent_factory()\n",
|
||||
"agent.invoke({\"question\": \"abc\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "89124d06-41f7-4432-9f2e-542c0d85e2e5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Benchmarking\n",
|
||||
"\n",
|
||||
"See `introduction` and `benchmark all` for information on how to run benchmarks. This notebook is just to here to explain and explore the task."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
```{toctree}
|
||||
:maxdepth: 2
|
||||
:caption: Introduction
|
||||
|
||||
./notebooks/getting_started
|
||||
./notebooks/models
|
||||
./notebooks/datasets
|
||||
```
|
||||
|
||||
|
||||
```{toctree}
|
||||
:maxdepth: 0
|
||||
:caption: Tool Usage
|
||||
|
||||
./notebooks/tool_usage/intro
|
||||
./notebooks/tool_usage/relational_data
|
||||
./notebooks/tool_usage/multiverse_math
|
||||
./notebooks/tool_usage/typewriter_1
|
||||
./notebooks/tool_usage/typewriter_26
|
||||
./notebooks/tool_usage/benchmark_all_tasks
|
||||
```
|
||||
|
||||
```{toctree}
|
||||
:maxdepth: 0
|
||||
:caption: Extraction
|
||||
|
||||
./notebooks/extraction/intro
|
||||
./notebooks/extraction/email
|
||||
./notebooks/extraction/chat_extraction
|
||||
./notebooks/extraction/high_cardinality
|
||||
```
|
||||
|
||||
```{toctree}
|
||||
:maxdepth: 2
|
||||
:caption: RAG
|
||||
|
||||
./notebooks/retrieval/intro
|
||||
./notebooks/retrieval/langchain_docs_qa
|
||||
./notebooks/retrieval/semi_structured_benchmarking/semi_structured
|
||||
./notebooks/retrieval/semi_structured_benchmarking/ss_eval_chunk_sizes
|
||||
./notebooks/retrieval/semi_structured_benchmarking/ss_eval_long_context
|
||||
./notebooks/retrieval/semi_structured_benchmarking/ss_eval_multi_vector
|
||||
./notebooks/retrieval/multi_modal_benchmarking/multi_modal_eval_baseline
|
||||
./notebooks/retrieval/multi_modal_benchmarking/multi_modal_eval
|
||||
./notebooks/retrieval/comparing_techniques
|
||||
```
|
||||
|
||||
```{toctree}
|
||||
:maxdepth: 2
|
||||
:caption: Benchmarking Without LangSmith
|
||||
./notebooks/run_without_langsmith
|
||||
```
|
||||
@@ -0,0 +1,76 @@
|
||||
import streamlit as st
|
||||
from langsmith import Client
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.chains import create_extraction_chain
|
||||
|
||||
st.set_page_config(page_title='🦜🔗 Text-to-graph extraction')
|
||||
client = Client()
|
||||
def send_feedback(run_id, score):
|
||||
client.create_feedback(run_id, "user_score", score=score)
|
||||
|
||||
st.title('🦜🔗 Text-to-graph playground')
|
||||
st.info("This playground explores the use of [OpenAI functions](https://openai.com/blog/function-calling-and-other-api-updates) and [LangChain](https://github.com/langchain-ai/langchain) to build knowledge graphs from user-input text. It breaks down the user input text into knowledge graph triples of subject (primary entities or concepts in a sentence), predicate (actions or relationships that connect subjects to objects), and object (entities or concepts that interact with or are acted upon by the subjects).")
|
||||
|
||||
# Input text (optional default)
|
||||
oppenheimer_text=''''Julius Robert Oppenheimer, often known as Robert or "Oppie", is heralded as the father of the atomic bomb. Emerging from a non-practicing Jewish family in New York, he made several breakthroughs, such as the early black hole theory, before the monumental Manhattan Project. His wife, Katherine “Kitty” Oppenheimer, was a German-born woman with a complex past, including connections to the Communist Party. Oppenheimer\'s journey was beset by political adversaries, notably Lewis Strauss, chairman of the U.S. Atomic Energy Commission, and William Borden, an executive director with hawkish nuclear ambitions. These tensions culminated in the famous 1954 security hearing. Influential figures like lieutenant general Leslie Groves, who had also overseen the Pentagon\'s creation, stood by Oppenheimer\'s side, having earlier chosen him for the Manhattan Project and the Los Alamos location. Intimate relationships, like that with Jean Tatlock, a Communist and the possible muse behind the Trinity test\'s name, and colleagues like Frank, Oppenheimer\'s physicist brother, intertwined with his professional life. Scientists such as Ernest Lawrence, Edward Teller, David Hill, Richard Feynman, and Hans Bethe were some of Oppenheimer\'s contemporaries, each contributing to and contesting the atomic age\'s directions. Boris Pash\'s investigations, and the perspectives of figures like Leo Szilard, Niels Bohr, Harry Truman, and others, framed the broader sociopolitical context. Meanwhile, individuals like Robert Serber, Enrico Fermi, Albert Einstein, and Isidor Isaac Rabi, among many others, each played their parts in this narrative, from naming the atomic bombs to pivotal scientific contributions and advisory roles. All these figures, together with the backdrop of World War II, McCarthyism, and the dawn of the nuclear age, presented a complex mosaic of ambitions, loyalties, betrayals, and ideologies.oppenheimer_short.txt'''
|
||||
|
||||
# Knowledge triplet schema
|
||||
default_schema = {
|
||||
"properties": {
|
||||
"subject": {"type": "string"},
|
||||
"predicate": {"type": "string"},
|
||||
"object": {"type": "string"},
|
||||
},
|
||||
"required": ["subject", "predicate", "object"],
|
||||
}
|
||||
|
||||
# Create a text_area, set the default value to oppenheimer_text
|
||||
MAX_CHARS = 2000 # Maximum number of characters
|
||||
user_input_text = st.text_area("Enter your text (<2000 characters):", height=200)
|
||||
if len(user_input_text) > MAX_CHARS:
|
||||
st.warning(f"Text is too long. Processing only the first {MAX_CHARS} characters")
|
||||
user_input_text = user_input_text[:MAX_CHARS]
|
||||
|
||||
# Output formatting of triples
|
||||
def json_to_markdown_table(json_list):
|
||||
if not json_list:
|
||||
return "No data available."
|
||||
|
||||
# Extract headers
|
||||
headers = json_list[0].keys()
|
||||
markdown_table = " | ".join(headers) + "\n"
|
||||
markdown_table += " | ".join(["---"] * len(headers)) + "\n"
|
||||
|
||||
# Extract rows
|
||||
for item in json_list:
|
||||
row = " | ".join([str(item[header]) for header in headers])
|
||||
markdown_table += row + "\n"
|
||||
|
||||
return markdown_table
|
||||
|
||||
# Form input and query
|
||||
markdown_output = None
|
||||
with st.form('myform', clear_on_submit=True):
|
||||
|
||||
submitted = st.form_submit_button('Submit')
|
||||
if submitted:
|
||||
|
||||
with st.spinner('Calculating...'):
|
||||
|
||||
llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo")
|
||||
chain = create_extraction_chain(default_schema, llm)
|
||||
extraction_output = chain(user_input_text,include_run_info=True)
|
||||
markdown_output = json_to_markdown_table(extraction_output['text'])
|
||||
run_id = extraction_output["__run"].run_id
|
||||
|
||||
# Feeback
|
||||
if markdown_output is not None:
|
||||
st.markdown(markdown_output)
|
||||
col_blank, col_text, col1, col2 = st.columns([10, 2,1,1])
|
||||
with col_text:
|
||||
st.text("Feedback:")
|
||||
with col1:
|
||||
st.button("👍", on_click=send_feedback, args=(run_id, 1))
|
||||
with col2:
|
||||
st.button("👎", on_click=send_feedback, args=(run_id, 0))
|
||||
|
||||
+2
-2
@@ -1,8 +1,9 @@
|
||||
from chat_langchain.chain import chain
|
||||
from fastapi import FastAPI
|
||||
from langserve import add_routes
|
||||
from chat_langchain.chain import chain
|
||||
from openai_functions_agent import agent_executor as openai_functions_agent_chain
|
||||
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
# Edit this to add the chain you want to add
|
||||
@@ -15,7 +16,6 @@ add_routes(
|
||||
|
||||
add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
|
||||
|
||||
|
||||
def run_server(port: int = 1983):
|
||||
import uvicorn
|
||||
|
||||
+1
-1
@@ -12,6 +12,6 @@ RETRIEVER_TOOL_NAME = "search"
|
||||
|
||||
|
||||
@tool
|
||||
def search(query, callbacks=None):
|
||||
def search(query, callbacks = None):
|
||||
"""Search the LangChain docs with the retriever."""
|
||||
return retriever.get_relevant_documents(query, callbacks=callbacks)
|
||||
+4
-3
@@ -2,7 +2,7 @@
|
||||
from operator import itemgetter
|
||||
from typing import Dict, List, Optional, Sequence
|
||||
|
||||
from langchain.chat_models import ChatAnthropic, ChatFireworks, ChatOpenAI
|
||||
from langchain.chat_models import ChatAnthropic, ChatOpenAI, ChatFireworks
|
||||
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder, PromptTemplate
|
||||
from langchain.schema import Document
|
||||
from langchain.schema.language_model import BaseLanguageModel
|
||||
@@ -18,6 +18,7 @@ from langchain.schema.runnable import (
|
||||
from langchain_docs_retriever.retriever import get_retriever
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
RESPONSE_TEMPLATE = """\
|
||||
You are an expert programmer and problem-solver, tasked with answering any question \
|
||||
about Langchain.
|
||||
@@ -134,7 +135,7 @@ def create_response_chain(
|
||||
]
|
||||
)
|
||||
|
||||
response_generator = (prompt | llm | StrOutputParser()).with_config(
|
||||
response_synthesizer = (prompt | llm | StrOutputParser()).with_config(
|
||||
run_name="GenerateResponse",
|
||||
)
|
||||
return (
|
||||
@@ -147,7 +148,7 @@ def create_response_chain(
|
||||
),
|
||||
}
|
||||
| _context
|
||||
| response_generator
|
||||
| response_synthesizer
|
||||
)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user