Compare commits

...

15 Commits

Author SHA1 Message Date
ccurme 34cd281494 benchmarks[major]: bump core to 0.3 (#211)
- Drop support for python 3.8
- Bump langchain-core to 0.3
- Update pydantic objects to v2
2024-10-21 16:47:14 -04:00
Isaac Francisco 99cf03a50a add faiss-cpu dependency (#209) 2024-08-07 07:53:45 -07:00
Isaac Francisco b36a339a65 Isaac/realfixes (#208) 2024-08-06 15:28:43 -07:00
Isaac Francisco 442cb47fc9 Isaac/realfixes (#207) 2024-08-06 15:24:23 -07:00
Isaac Francisco b7795c7df1 change wd (#206) 2024-08-06 15:15:08 -07:00
Isaac Francisco ac161de968 thanks erick (#205) 2024-08-06 14:50:39 -07:00
Isaac Francisco d91944bb07 test (#204) 2024-08-06 14:45:48 -07:00
Isaac Francisco 8798bd3105 test (#203) 2024-08-06 14:40:01 -07:00
Isaac Francisco 621eea5d93 Isaac/tryingpoetryagain (#202) 2024-08-06 14:36:43 -07:00
Isaac Francisco b6590a8745 Isaac/changepoetry (#201) 2024-08-06 14:30:42 -07:00
Isaac Francisco 458ffa70ea test (#200) 2024-08-06 14:26:56 -07:00
Isaac Francisco ebe5c117c2 test (#198) 2024-08-06 14:14:39 -07:00
Ikko Eltociear Ashimine adff80af11 docs: update README.md (#195)
Mutiverse -> Multiverse
2024-07-24 11:13:42 -07:00
Bagatur 301837e303 Release 0.0.14 (#194) 2024-07-24 08:00:17 -07:00
Bagatur 4f1d922a6e minor: bump to langchain v2 (#191) 2024-07-24 07:59:19 -07:00
33 changed files with 1722 additions and 2351 deletions
@@ -1,94 +0,0 @@
name: pydantic v1/v2 compatibility
on:
workflow_call:
inputs:
working-directory:
required: true
type: string
description: "From which folder this pipeline executes"
env:
POETRY_VERSION: "1.6.1"
jobs:
build:
timeout-minutes: 5
defaults:
run:
working-directory: ${{ inputs.working-directory }}
runs-on: ubuntu-latest
strategy:
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10"
- "3.11"
name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
python-version: ${{ matrix.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
working-directory: ${{ inputs.working-directory }}
cache-key: pydantic-cross-compat
- name: Install dependencies
shell: bash
run: poetry install --with test
- name: Install the opposite major version of pydantic
# If normal tests use pydantic v1, here we'll use v2, and vice versa.
shell: bash
run: |
# Determine the major part of pydantic version
REGULAR_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
if [[ "$REGULAR_VERSION" == "1" ]]; then
PYDANTIC_DEP=">=2.1,<3"
TEST_WITH_VERSION="2"
elif [[ "$REGULAR_VERSION" == "2" ]]; then
PYDANTIC_DEP="<2"
TEST_WITH_VERSION="1"
else
echo "Unexpected pydantic major version '$REGULAR_VERSION', cannot determine which version to use for cross-compatibility test."
exit 1
fi
# Install via `pip` instead of `poetry add` to avoid changing lockfile,
# which would prevent caching from working: the cache would get saved
# to a different key than where it gets loaded from.
poetry run pip install "pydantic${PYDANTIC_DEP}"
# Ensure that the correct pydantic is installed now.
echo "Checking pydantic version... Expecting ${TEST_WITH_VERSION}"
# Determine the major part of pydantic version
CURRENT_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
# Check that the major part of pydantic version is as expected, if not
# raise an error
if [[ "$CURRENT_VERSION" != "$TEST_WITH_VERSION" ]]; then
echo "Error: expected pydantic version ${CURRENT_VERSION} to have been installed, but found: ${TEST_WITH_VERSION}"
exit 1
fi
echo "Found pydantic version ${CURRENT_VERSION}, as expected"
- name: Run pydantic compatibility tests
shell: bash
run: make test
- name: Ensure the tests did not create any additional files
shell: bash
run: |
set -eu
STATUS="$(git status)"
echo "$STATUS"
# grep will exit non-zero if the target message isn't found,
# and `set -e` above will cause the step to fail.
echo "$STATUS" | grep 'nothing to commit, working tree clean'
-6
View File
@@ -31,12 +31,6 @@ jobs:
working-directory: .
secrets: inherit
pydantic-compatibility:
uses:
./.github/workflows/_pydantic_compatibility.yml
with:
working-directory: .
secrets: inherit
test:
timeout-minutes: 5
runs-on: ubuntu-latest
+13 -2
View File
@@ -5,6 +5,12 @@ on:
schedule:
- cron: '0 0 * * 0' # Runs at midnight (00:00) every Sunday (UTC time)
env:
POETRY_VERSION: "1.6.1"
LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
jobs:
run_tool_benchmarks:
runs-on: ubuntu-latest
@@ -27,7 +33,12 @@ jobs:
poetry install --with test,lint,typing,docs
- name: Multiverse math benchmark
run: python scripts/multiverse_math_benchmark.py
run: |
cd scripts
poetry run python multiverse_math_benchmark.py
- name: Query analysis benchmark
run: python scripts/query_analysis_benchmark.py
run: |
cd scripts
poetry run python query_analysis_benchmark.py
+1 -1
View File
@@ -43,7 +43,7 @@ Explore Agent Traces on LangSmith:
* [Relational Data](https://smith.langchain.com/public/22721064-dcf6-4e42-be65-e7c46e6835e7/d)
* [Tool Usage (1-tool)](https://smith.langchain.com/public/ac23cb40-e392-471f-b129-a893a77b6f62/d)
* [Tool Usage (26-tools)](https://smith.langchain.com/public/366bddca-62b3-4b6e-849b-a478abab73db/d)
* [Mutiverse Math](https://smith.langchain.com/public/983faff2-54b9-4875-9bf2-c16913e7d489/d)
* [Multiverse Math](https://smith.langchain.com/public/983faff2-54b9-4875-9bf2-c16913e7d489/d)
## Installation
+1 -1
View File
@@ -3,12 +3,12 @@ from langchain.agents import AgentExecutor, OpenAIFunctionsAgent
from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
create_retriever_tool,
)
from langchain.chat_models import ChatOpenAI
from langchain.embeddings import OpenAIEmbeddings
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
from langchain.smith import RunEvalConfig, run_on_dataset
from langchain.tools import PythonAstREPLTool
from langchain.vectorstores import FAISS
from langchain_openai import ChatOpenAI
from langsmith import Client
from pydantic import BaseModel, Field
+1 -1
View File
@@ -1,8 +1,8 @@
import pandas as pd
from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
from langchain.agents.agent_types import AgentType
from langchain.chat_models import ChatOpenAI
from langchain.smith import RunEvalConfig, run_on_dataset
from langchain_openai import ChatOpenAI
from langsmith import Client
if __name__ == "__main__":
+1 -1
View File
@@ -1,8 +1,8 @@
import pandas as pd
from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
from langchain.agents.agent_types import AgentType
from langchain.chat_models import ChatOpenAI
from langchain.smith import RunEvalConfig, run_on_dataset
from langchain_openai import ChatOpenAI
from langsmith import Client
if __name__ == "__main__":
+1 -1
View File
@@ -1,8 +1,8 @@
import pandas as pd
from langchain.chat_models import ChatOpenAI
from langchain.prompts import ChatPromptTemplate
from langchain.schema.output_parser import StrOutputParser
from langchain.smith import RunEvalConfig, run_on_dataset
from langchain_openai import ChatOpenAI
from langsmith import Client
from pandasai import PandasAI
+1 -1
View File
@@ -2,7 +2,7 @@ import pandas as pd
import streamlit as st
from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
from langchain.agents.agent_types import AgentType
from langchain.chat_models import ChatOpenAI
from langchain_openai import ChatOpenAI
df = pd.read_csv("titanic.csv")
+1 -1
View File
@@ -1,6 +1,6 @@
import streamlit as st
from langchain.chains import create_extraction_chain
from langchain.chat_models import ChatOpenAI
from langchain_openai import ChatOpenAI
from langsmith import Client
st.set_page_config(page_title="🦜🔗 Text-to-graph extraction")
@@ -1,8 +1,8 @@
from langchain.chat_models import ChatAnthropic
from langchain.prompts import ChatPromptTemplate
from langchain.pydantic_v1 import BaseModel
from langchain.schema.output_parser import StrOutputParser
from langchain.schema.runnable import RunnableLambda
from pydantic import BaseModel
from .prompts import answer_prompt
from .retriever_agent import executor
@@ -3,13 +3,13 @@ from typing import List, Tuple
from langchain.agents import AgentExecutor
from langchain.agents.format_scratchpad import format_to_openai_functions
from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
from langchain.chat_models import ChatOpenAI
from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
from langchain.pydantic_v1 import BaseModel, Field
from langchain.schema.messages import AIMessage, HumanMessage
from langchain.tools import tool
from langchain.tools.render import format_tool_to_openai_function
from langchain_docs_retriever.retriever import get_retriever
from langchain_openai import ChatOpenAI
from pydantic import BaseModel, Field
# This is used to tell the model how to best use the retriever.
@@ -7,9 +7,9 @@ from typing import Callable, Optional
from anthropic_iterative_search.chain import chain as anthropic_agent_chain
from chat_langchain.chain import create_chain
from langchain.chat_models import ChatOpenAI
from langchain.schema.runnable import Runnable
from langchain.smith import RunEvalConfig, run_on_dataset
from langchain_openai import ChatOpenAI
from langsmith import Client
from oai_assistant.chain import agent_executor as openai_assistant_chain
from openai_functions_agent import agent_executor as openai_functions_agent_chain
@@ -259,8 +259,8 @@
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"llm = ChatOpenAI(model=\"gpt-4-1106-preview\", temperature=0).bind_functions(\n",
" functions=[task.schema],\n",
+1 -1
View File
@@ -232,8 +232,8 @@
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-16k\", temperature=0).bind_functions(\n",
" functions=[task.schema],\n",
+1 -1
View File
@@ -97,7 +97,7 @@
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"from langchain_benchmarks.extraction import get_eval_config\n",
"\n",
@@ -75,6 +75,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "7fb27b941602401d91542211134fc71a",
"metadata": {},
"outputs": [],
"source": [
@@ -728,12 +729,12 @@
"from langchain.agents import AgentExecutor\n",
"from langchain.agents.format_scratchpad import format_to_openai_functions\n",
"from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
"from langchain.pydantic_v1 import BaseModel, Field\n",
"from langchain.schema.messages import AIMessage, HumanMessage\n",
"from langchain.tools import tool\n",
"from langchain.tools.render import format_tool_to_openai_function\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"# This is used to tell the model how to best use the retriever.\n",
"\n",
@@ -508,8 +508,8 @@
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.schema.messages import HumanMessage\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"\n",
"def image_summarize(img_base64, prompt):\n",
@@ -328,10 +328,10 @@
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts import ChatPromptTemplate\n",
"from langchain.schema.output_parser import StrOutputParser\n",
"from langchain.schema.runnable import RunnablePassthrough\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"\n",
"def rag_chain(retriever):\n",
@@ -451,11 +451,11 @@
"source": [
"from operator import itemgetter\n",
"\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts import ChatPromptTemplate\n",
"from langchain.schema.document import Document\n",
"from langchain.schema.output_parser import StrOutputParser\n",
"from langchain.schema.runnable.passthrough import RunnableAssign\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"# Prompt\n",
"prompt = ChatPromptTemplate.from_messages(\n",
@@ -126,7 +126,6 @@
"source": [
"import uuid\n",
"\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.document_loaders import PyPDFLoader\n",
"from langchain.embeddings import OpenAIEmbeddings\n",
"from langchain.prompts import ChatPromptTemplate\n",
@@ -138,6 +137,7 @@
"from langchain.storage import InMemoryStore\n",
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"from langchain.vectorstores import Chroma\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"\n",
"def prepare_documents(docs):\n",
@@ -1,8 +1,8 @@
from typing import Optional
from langchain.chat_models import ChatOpenAI
from langchain.chat_models.base import BaseChatModel
from langchain.smith import RunEvalConfig
from langchain_openai import ChatOpenAI
def get_eval_config(eval_llm: Optional[BaseChatModel] = None) -> RunEvalConfig:
@@ -2,10 +2,10 @@
from typing import Any, Dict, List, Optional, Type
from langchain.chains.openai_functions import convert_to_openai_function
from langchain.chat_models import ChatOpenAI
from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser
from langchain.prompts import ChatPromptTemplate
from langchain.schema.runnable import Runnable
from langchain_openai import ChatOpenAI
from langsmith.client import Client
from pydantic import BaseModel
@@ -1,7 +1,7 @@
from enum import Enum
from typing import List, Optional
from langchain.pydantic_v1 import BaseModel, Field
from pydantic import BaseModel, Field
class QuestionCategory(str, Enum):
@@ -2,7 +2,7 @@ from enum import Enum
from typing import List, Optional
from langchain.prompts import ChatPromptTemplate
from langchain.pydantic_v1 import BaseModel, Field
from pydantic import BaseModel, Field
from langchain_benchmarks.schema import ExtractionTask
@@ -1,7 +1,7 @@
from langchain.smith import RunEvalConfig
from langchain_core.pydantic_v1 import BaseModel, Field
from langsmith.evaluation import EvaluationResult, run_evaluator
from langsmith.schemas import Example, Run
from pydantic import BaseModel, Field
from langchain_benchmarks.schema import ExtractionTask
+1 -1
View File
@@ -1,8 +1,8 @@
from typing import Optional
from langchain.chat_models import ChatOpenAI
from langchain.evaluation import load_evaluator
from langchain.smith import RunEvalConfig
from langchain_openai import ChatOpenAI
try:
from langchain.schema.language_model import BaseLanguageModel
@@ -1,9 +1,9 @@
from typing import Optional
from langchain.base_language import BaseLanguageModel
from langchain.chat_models import ChatOpenAI
from langchain.schema.retriever import BaseRetriever
from langchain.schema.runnable import Runnable
from langchain_openai import ChatOpenAI
from langchain_benchmarks.rag.tasks.langchain_docs.architectures.crqa import (
create_response_chain,
+2 -2
View File
@@ -3,11 +3,9 @@ import os
from functools import partial
from typing import Callable, Iterable, List, Optional
from langchain.chat_models import ChatOpenAI
from langchain.indexes import SQLRecordManager, index
from langchain.output_parsers.openai_functions import JsonKeyOutputFunctionsParser
from langchain.prompts import ChatPromptTemplate
from langchain.pydantic_v1 import BaseModel
from langchain.retrievers.multi_vector import MultiVectorRetriever
from langchain.retrievers.parent_document_retriever import ParentDocumentRetriever
from langchain.schema.document import Document
@@ -18,6 +16,8 @@ from langchain.schema.storage import BaseStore
from langchain.schema.vectorstore import VectorStore
from langchain.storage import InMemoryStore
from langchain.text_splitter import RecursiveCharacterTextSplitter, TextSplitter
from langchain_openai import ChatOpenAI
from pydantic import BaseModel
from tqdm.auto import tqdm
logger = logging.getLogger(__name__)
@@ -10,11 +10,11 @@ from typing import Any, Literal, Optional, Union
from langchain.callbacks.manager import collect_runs
from langchain.chains import LLMChain
from langchain.chat_models import ChatOpenAI
from langchain.evaluation import EvaluatorType, StringEvaluator, load_evaluator
from langchain.evaluation.schema import StringEvaluator
from langchain.smith import RunEvalConfig
from langchain_core.language_models import BaseChatModel, BaseLanguageModel
from langchain_openai import ChatOpenAI
from langsmith.evaluation.evaluator import (
EvaluationResult,
EvaluationResults,
@@ -175,7 +175,7 @@ class AgentTrajectoryEvaluator(RunEvaluator):
eval_llm = eval_llm or ChatOpenAI(
model="gpt-4",
temperature=0,
model_kwargs={"seed": 42},
seed=42,
max_retries=1,
request_timeout=60,
)
@@ -1,10 +1,10 @@
from datetime import datetime
from typing import List, Literal, Union, cast
from langchain.pydantic_v1 import BaseModel, Field
from langchain.tools import BaseTool, tool
from langchain_core.messages import HumanMessage
from langsmith.client import Client
from pydantic import BaseModel, Field
from langchain_benchmarks.schema import ToolUsageEnvironment, ToolUsageTask
Generated
+1665 -2188
View File
File diff suppressed because it is too large Load Diff
+12 -30
View File
@@ -1,49 +1,33 @@
[tool.poetry]
name = "langchain-benchmarks"
version = "0.0.13"
version = "0.0.15"
description = "🦜💪 Flex those feathers!"
authors = ["LangChain AI"]
license = "MIT"
readme = "README.md"
[tool.poetry.dependencies]
python = "^3.8.1"
langchain = "^0.1.15"
python = "^3.9"
langchain = "^0.3"
langchain-community = "^0.3"
langchain-core= "^0.3.12"
langsmith = ">=0.0.70"
tqdm = "^4"
ipywidgets = "^8"
tabulate = ">=0.8.0"
langchain-openai = "^0.2"
[tool.poetry.group.dev]
optional = true
[tool.poetry.group.dev.dependencies]
jupyter = "^1.0.0"
langchain-core = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/core"}
langchain = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/langchain"}
langchain-anthropic = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/anthropic"}
langchain-google-vertexai= {git = "https://github.com/langchain-ai/langchain-google.git", subdirectory = "libs/vertexai/"}
langchain-fireworks = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/fireworks"}
langchain-mistralai = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/mistralai"}
langchain-cohere = {git = "https://github.com/langchain-ai/langchain-cohere.git", subdirectory="libs/cohere"}
langchain-groq = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/groq"}
langchain-openai = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/openai"}
[tool.poetry.group.typing]
optional = true
[tool.poetry.group.typing.dependencies]
mypy = "^1.7.0"
langchain-core = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/core"}
langchain = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/langchain"}
langchain-anthropic = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/anthropic"}
langchain-fireworks = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/fireworks"}
langchain-mistralai = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/mistralai"}
langchain-cohere = {git = "https://github.com/langchain-ai/langchain-cohere.git", subdirectory="libs/cohere"}
langchain-groq = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/groq"}
langchain-openai = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/openai"}
[tool.poetry.group.lint]
optional = true
@@ -74,14 +58,12 @@ pytest-socket = "^0.6.0"
pytest-watch = "^4.2.0"
pytest-timeout = "^2.2.0"
freezegun = "^1.3.1"
langchain-core = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/core"}
langchain = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/langchain"}
langchain-anthropic = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/anthropic"}
langchain-fireworks = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/fireworks"}
langchain-mistralai = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/mistralai"}
langchain-cohere = {git = "https://github.com/langchain-ai/langchain-cohere.git", subdirectory="libs/cohere"}
langchain-groq = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/groq"}
langchain-openai = {git = "https://github.com/langchain-ai/langchain.git", subdirectory = "libs/partners/openai"}
langchain-anthropic = "^0.2"
langchain-fireworks = "^0.2"
langchain-mistralai = "^0.2"
langchain-groq = "^0.2"
langchain-core = "^0.3.12"
faiss-cpu = ">=1.8.0"
[tool.ruff]
select = [