infra: release permissions (#193)

Co-authored-by: Bagatur <baskaryan@gmail.com>

.github/workflows/release.yml

@@ -8,6 +8,7 @@ jobs:
   release:
     uses:
       ./.github/workflows/_release.yml
+    permissions: write-all
     with:
       working-directory: .
     secrets: inherit

.github/workflows/tool_benchmarks.yml

@@ -0,0 +1,33 @@
+name: Weekly Tool Benchmarks
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 0'  # Runs at midnight (00:00) every Sunday (UTC time)
+
+jobs:
+  run_tool_benchmarks:
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12 + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: '3.12'
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+      
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Multiverse math benchmark
+        run: python scripts/multiverse_math_benchmark.py
+      
+      - name: Query analysis benchmark
+        run: python scripts/query_analysis_benchmark.py

langchain_benchmarks/model_registration.py

@@ -5,7 +5,7 @@ from langchain_benchmarks.schema import ModelRegistry, RegisteredModel
 _OPEN_AI_MODELS = [
     RegisteredModel(
         provider="openai",
-        name="gpt-3.5-turbo-0125",
+        name="gpt-3.5-turbo-1106",
         type="chat",
         description=(
             "The latest GPT-3.5 Turbo model with improved instruction following, "
@@ -13,7 +13,7 @@ _OPEN_AI_MODELS = [
             "Returns a maximum of 4,096 output tokens."
         ),
         params={
-            "model": "gpt-3.5-turbo-0125",
+            "model": "gpt-3.5-turbo-1106",
         },
     ),
     RegisteredModel(
@@ -162,27 +162,9 @@ _OPEN_AI_MODELS = [
             "model": "gpt-4-32k-0314",
         },
     ),
-    RegisteredModel(
-        provider="openai",
-        name="gpt-4o",
-        description="GPT-4o base model",
-        type="chat",
-        params={
-            "model": "gpt-4o",
-        },
-    ),
 ]
 
 _FIREWORKS_MODELS = [
-    RegisteredModel(
-        provider="fireworks",
-        name="firefunction-v2",
-        type="chat",
-        description="Fireworks function calling model",
-        params={
-            "model": "accounts/fireworks/models/firefunction-v2",
-        }, 
-    ),
     RegisteredModel(
         provider="fireworks",
         name="llama-v2-7b-chat-fw",
@@ -225,20 +207,11 @@ _FIREWORKS_MODELS = [
         description="Mistral MoE 8x7B Instruct v0.1 model with Sparse "
         "Mixture of Experts. Fine tuned for instruction following",
         type="llm",
-        params={
-            "model": "accounts/fireworks/models/mixtral-8x7b-instruct"
-        },
+        params={"model": "accounts/fireworks/models/mixtral-8x7b-instruct"},
     ),
 ]
 
 _ANTHROPIC_MODELS = [
-    RegisteredModel(
-        provider="anthropic",
-        name="claude-3-5-sonnet-20240620",
-        description="Highest performing Claude model as of July 2024",
-        type="chat",
-        params={"model": "claude-3-5-sonnet-20240620"},
-    ),
     RegisteredModel(
         provider="anthropic",
         name="claude-3-haiku-20240307",
@@ -325,7 +298,6 @@ _ANYSCALE_MODELS = [
     ),
 ]
 
-
 model_registry = ModelRegistry(
     registered_models=_OPEN_AI_MODELS
     + _FIREWORKS_MODELS

langchain_benchmarks/tool_usage/prompts.py

@@ -33,7 +33,9 @@ INPUT_A: input_a here
 INPUT_B: input_b here
 COMPARISON: CORRECT or INCORRECT here
 
-Ignore differences in punctuation and phrasing between the student answer and true answer. 
+Ignore differences in punctuation and phrasing between the student answer and true answer, please only compare the first 4 decimal digits.
+
+For instance if INPUT_A = 123.6751345 and INPUT_B = 123.6751456 you should return CORRECT, since the first 4 decimal points match.
 
 Begin!
 

langchain_benchmarks/tool_usage/tasks/query_analysis.py

@@ -4,8 +4,11 @@ from typing import List, Literal, Union, cast
 from langchain.pydantic_v1 import BaseModel, Field
 from langchain.tools import BaseTool, tool
 from langchain_core.messages import HumanMessage
+from langsmith.client import Client
 
 from langchain_benchmarks.schema import ToolUsageEnvironment, ToolUsageTask
+
+
 class DocQuery(BaseModel):
     """Query against documentation"""
 
@@ -13,7 +16,7 @@ class DocQuery(BaseModel):
     source: Literal["langchain", "langsmith", "langgraph"] = Field(
         ...,
         description="The documentation source to search against. Should be one of 'langchain', 'langsmith', or "
-        "'langgraph' depending on which one product the user question pertains to"
+        "'langgraph' depending on which one product the user question pertains to",
     )
 
 
@@ -69,7 +72,11 @@ def get_environment() -> ToolUsageEnvironment:
 
 DOC_DATASET = [
     {
-        "question": [HumanMessage("Can I use the send method to map-reduce the values of different branch points?")],
+        "question": [
+            HumanMessage(
+                "Can I use the send method to map-reduce the values of different branch points?"
+            )
+        ],
         "tool_calls": [
             {
                 "name": "DocQuery",
@@ -103,7 +110,7 @@ DOC_DATASET = [
                     "start_date": None,
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
     {
@@ -113,20 +120,24 @@ DOC_DATASET = [
         "tool_calls": [
             {
                 "name": "DocQuery",
-                "args": {"query": "pairwise evals different models", "source": "langsmith"},
+                "args": {
+                    "query": "pairwise evals different models",
+                    "source": "langsmith",
+                },
             }
         ],
     },
     {
         "question": [HumanMessage("Can a user update state during a run?")],
         "tool_calls": [
-            {"name": "DocQuery", "args": {"query": "user update state", "source": "langgraph"}}
+            {
+                "name": "DocQuery",
+                "args": {"query": "user update state", "source": "langgraph"},
+            }
         ],
     },
     {
-        "question": [
-            HumanMessage("Can I change config after each AI response?")
-        ],
+        "question": [HumanMessage("Can I change config after each AI response?")],
         "tool_calls": [
             {
                 "name": "DocQuery",
@@ -135,7 +146,11 @@ DOC_DATASET = [
         ],
     },
     {
-        "question": [HumanMessage("How can I build my own run rules? Can I set up a schedule for them?")],
+        "question": [
+            HumanMessage(
+                "How can I build my own run rules? Can I set up a schedule for them?"
+            )
+        ],
         "tool_calls": [
             {
                 "name": "DocQuery",
@@ -144,7 +159,7 @@ DOC_DATASET = [
             {
                 "name": "DocQuery",
                 "args": {"query": "run rules schedule", "source": "langsmith"},
-            }
+            },
         ],
     },
     {
@@ -176,15 +191,11 @@ DOC_DATASET = [
                     "start_date": None,
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
     {
-        "question": [
-            HumanMessage(
-                "is it possible to prevent exposing personal data?"
-            )
-        ],
+        "question": [HumanMessage("is it possible to prevent exposing personal data?")],
         "tool_calls": [
             {
                 "name": "DocQuery",
@@ -193,11 +204,7 @@ DOC_DATASET = [
         ],
     },
     {
-        "question": [
-            HumanMessage(
-                "How do you use conditional entry?"
-            )
-        ],
+        "question": [HumanMessage("How do you use conditional entry?")],
         "tool_calls": [
             {
                 "name": "DocQuery",
@@ -218,7 +225,10 @@ DOC_DATASET = [
             },
             {
                 "name": "DocQuery",
-                "args": {"query": "combine image and text in a prompt", "source": "langchain"},
+                "args": {
+                    "query": "combine image and text in a prompt",
+                    "source": "langchain",
+                },
             },
         ],
     },
@@ -231,7 +241,10 @@ DOC_DATASET = [
         "tool_calls": [
             {
                 "name": "DocQuery",
-                "args": {"query": "automation rules for chat model app", "source": "langsmith"},
+                "args": {
+                    "query": "automation rules for chat model app",
+                    "source": "langsmith",
+                },
             },
             {
                 "name": "DocQuery",
@@ -241,9 +254,7 @@ DOC_DATASET = [
     },
     {
         "question": [
-            HumanMessage(
-                "where can I read about how use Chroma embeddings locally?"
-            )
+            HumanMessage("where can I read about how use Chroma embeddings locally?")
         ],
         "tool_calls": [
             {
@@ -258,15 +269,11 @@ DOC_DATASET = [
                     "start_date": None,
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
     {
-        "question": [
-            HumanMessage(
-                "how to index documents in a RAG app?"
-            )
-        ],
+        "question": [HumanMessage("how to index documents in a RAG app?")],
         "tool_calls": [
             {
                 "name": "DocQuery",
@@ -277,7 +284,7 @@ DOC_DATASET = [
                 "args": {"query": "index documents RAG app", "source": "langgraph"},
             },
         ],
-    }
+    },
 ]
 
 TWEET_DATASET = [
@@ -347,7 +354,7 @@ TWEET_DATASET = [
                     "start_date": datetime(2023, 1, 1),
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
     {
@@ -372,7 +379,7 @@ TWEET_DATASET = [
                     "start_date": datetime(2023, 4, 1),
                     "end_date": datetime(2023, 6, 30),
                 },
-            }
+            },
         ],
     },
     {
@@ -460,14 +467,12 @@ BLOG_DATASET = [
                     "end_date": datetime(2023, 11, 30),
                     "has_link": False,
                 },
-            }
+            },
         ],
     },
     {
         "question": [
-            HumanMessage(
-                "what has been said about universal configurable models?"
-            )
+            HumanMessage("what has been said about universal configurable models?")
         ],
         "tool_calls": [
             {
@@ -489,7 +494,7 @@ BLOG_DATASET = [
                     "end_date": None,
                     "has_link": False,
                 },
-            }
+            },
         ],
     },
     {
@@ -529,11 +534,7 @@ BLOG_DATASET = [
         ],
     },
     {
-        "question": [
-            HumanMessage(
-                "Why is using fewshot prompting helpful?"
-            )
-        ],
+        "question": [HumanMessage("Why is using fewshot prompting helpful?")],
         "tool_calls": [
             {
                 "name": "BlogQuery",
@@ -568,7 +569,7 @@ BLOG_DATASET = [
             }
         ],
     },
-]# Realease notes/announcements + Case studies + 
+]  # Realease notes/announcements + Case studies +
 
 AMBIGUOUS_DATASET = [
     {
@@ -617,7 +618,10 @@ AMBIGUOUS_DATASET = [
         "tool_calls": [
             {
                 "name": "DocQuery",
-                "args": {"query": "document loader for RAG chain", "source": "langchain"},
+                "args": {
+                    "query": "document loader for RAG chain",
+                    "source": "langchain",
+                },
             },
             {
                 "name": "BlogQuery",
@@ -627,15 +631,11 @@ AMBIGUOUS_DATASET = [
                     "start_date": None,
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
     {
-        "question": [
-            HumanMessage(
-                "case studies using langgraph last week?"
-            )
-        ],
+        "question": [HumanMessage("case studies using langgraph last week?")],
         "tool_calls": [
             {
                 "name": "BlogQuery",
@@ -648,16 +648,16 @@ AMBIGUOUS_DATASET = [
             }
         ],
     },
-    
 ]
 
 DATASET = DOC_DATASET + TWEET_DATASET + BLOG_DATASET + AMBIGUOUS_DATASET
 
-EXTRACTION_TASK = ToolUsageTask(
+QUERY_ANALYSIS_TASK = ToolUsageTask(
     name="Extraction Task",
     dataset_id="https://smith.langchain.com/public/594f9f60-30a0-49bf-b075-f44beabf546a/d",
     create_environment=get_environment,
-    instructions=("""
+    instructions=(
+        """
                     You are requested to generate queries for searching either through tweets, docs, or blog entries. 
                     Inside the docs there are three different sources that you may wish to query for: LangGraph, LangSmith, or LangChain. 
                     LangGraph is a library for building multi-actor applications with LLMs, used to create agent and multi-agent workflows. 
@@ -675,7 +675,7 @@ An environment that contains three different mock query tools for searching thro
 The three tools are for querying LangChain documentation, tweets, and blogs respectively.
 
 The objective of the task it to measure how well the agent can select the correct tool and \
-select the right parameterse for the query. It is not a test of the actual querying process, \
+select the right parameters for the query. It is not a test of the actual querying process, \
 merely the process of constructing the query.
 """
     ),
@@ -704,14 +704,12 @@ FEW_SHOT_DATASET = [
                     "start_date": None,
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
     {
         "question": [
-            HumanMessage(
-                "How do you build a RAG chain with a Postgres vectorstore?"
-            )
+            HumanMessage("How do you build a RAG chain with a Postgres vectorstore?")
         ],
         "tool_calls": [
             {
@@ -725,15 +723,16 @@ FEW_SHOT_DATASET = [
             },
             {
                 "name": "DocQuery",
-                "args": {"query": "RAG chain with Postgres vectorstore", "source": "langchain"},
+                "args": {
+                    "query": "RAG chain with Postgres vectorstore",
+                    "source": "langchain",
+                },
             },
         ],
     },
     {
         "question": [
-            HumanMessage(
-                "What case studies have we written about tool usage?"
-            )
+            HumanMessage("What case studies have we written about tool usage?")
         ],
         "tool_calls": [
             {
@@ -748,19 +747,21 @@ FEW_SHOT_DATASET = [
         ],
     },
     {
-        "question": [
-            HumanMessage(
-                "How do I migrate from run_on_dataset to evaluate?"
-            )
-        ],
+        "question": [HumanMessage("How do I migrate from run_on_dataset to evaluate?")],
         "tool_calls": [
             {
                 "name": "DocQuery",
-                "args": {"query": "migrate run_on_dataset to evaluate", "source": "langchain"},
+                "args": {
+                    "query": "migrate run_on_dataset to evaluate",
+                    "source": "langchain",
+                },
             },
             {
                 "name": "DocQuery",
-                "args": {"query": "migrate run_on_dataset to evaluate", "source": "langsmith"},
+                "args": {
+                    "query": "migrate run_on_dataset to evaluate",
+                    "source": "langsmith",
+                },
             },
         ],
     },
@@ -777,7 +778,7 @@ FEW_SHOT_DATASET = [
                     "subject": "Anthropic",
                     "min_likes": None,
                     "max_likes": 100,
-                    "start_date": datetime(2023,11,1),
+                    "start_date": datetime(2023, 11, 1),
                     "end_date": None,
                     "has_link": True,
                 },
@@ -810,7 +811,7 @@ FEW_SHOT_DATASET = [
                     "end_date": None,
                     "has_link": False,
                 },
-            }
+            },
         ],
     },
     {
@@ -826,7 +827,7 @@ FEW_SHOT_DATASET = [
                     "subject": "filtering traces by metadata",
                     "authors": None,
                     "start_date": None,
-                    "end_date": datetime(2023,9,30),
+                    "end_date": datetime(2023, 9, 30),
                 },
             },
             {
@@ -836,14 +837,18 @@ FEW_SHOT_DATASET = [
                     "min_likes": None,
                     "max_likes": None,
                     "start_date": None,
-                    "end_date": datetime(2023,9,30),
+                    "end_date": datetime(2023, 9, 30),
                     "has_link": False,
                 },
-            }
+            },
         ],
     },
     {
-        "question": [HumanMessage("What updates to mistral partner package were posted in the last year?")],
+        "question": [
+            HumanMessage(
+                "What updates to mistral partner package were posted in the last year?"
+            )
+        ],
         "tool_calls": [
             {
                 "name": "TweetQuery",
@@ -851,7 +856,7 @@ FEW_SHOT_DATASET = [
                     "subject": "mistral partner package",
                     "min_likes": None,
                     "max_likes": None,
-                    "start_date": datetime(2023,1,1),
+                    "start_date": datetime(2023, 1, 1),
                     "end_date": None,
                     "has_link": False,
                 },
@@ -871,7 +876,7 @@ FEW_SHOT_DATASET = [
                     "subject": "best practices for initializing chat models",
                     "min_likes": None,
                     "max_likes": None,
-                    "start_date": datetime(2023,12,1),
+                    "start_date": datetime(2023, 12, 1),
                     "end_date": None,
                     "has_link": False,
                 },
@@ -881,31 +886,48 @@ FEW_SHOT_DATASET = [
                 "args": {
                     "subject": "best practices for initializing chat models",
                     "authors": None,
-                    "start_date": datetime(2023,12,1),
+                    "start_date": datetime(2023, 12, 1),
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
     {
-        "question": [HumanMessage("How can I learn about the differences between chat agents and graphs")],
+        "question": [
+            HumanMessage(
+                "How can I learn about the differences between chat agents and graphs"
+            )
+        ],
         "tool_calls": [
             {
                 "name": "DocQuery",
-                "args": {"query": "differences between chat agents and graphs", "source": "langchain"},
+                "args": {
+                    "query": "differences between chat agents and graphs",
+                    "source": "langchain",
+                },
             },
             {
                 "name": "DocQuery",
-                "args": {"query": "differences between chat agents and graphs", "source": "langgraph"},
-            }
+                "args": {
+                    "query": "differences between chat agents and graphs",
+                    "source": "langgraph",
+                },
+            },
         ],
     },
     {
-        "question": [HumanMessage("What are good practices to follow for switching from legacy packages?")],
+        "question": [
+            HumanMessage(
+                "What are good practices to follow for switching from legacy packages?"
+            )
+        ],
         "tool_calls": [
             {
                 "name": "DocQuery",
-                "args": {"query": "switching from legacy packages", "source": "langchain"},
+                "args": {
+                    "query": "switching from legacy packages",
+                    "source": "langchain",
+                },
             },
             {
                 "name": "BlogQuery",
@@ -915,7 +937,7 @@ FEW_SHOT_DATASET = [
                     "start_date": None,
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
     {
@@ -923,7 +945,10 @@ FEW_SHOT_DATASET = [
         "tool_calls": [
             {
                 "name": "DocQuery",
-                "args": {"query": "data exposed running custom evaluation", "source": "langsmith"},
+                "args": {
+                    "query": "data exposed running custom evaluation",
+                    "source": "langsmith",
+                },
             },
         ],
     },
@@ -953,34 +978,19 @@ FEW_SHOT_DATASET = [
                     "start_date": None,
                     "end_date": None,
                 },
-            }
+            },
         ],
     },
 ]
 
 
-
-def _create_dataset() -> None:
+def _create_dataset(examples: list, dataset_id: str) -> None:
     """Create a dataset with the langsmith client."""
-    from langsmith.client import Client
 
     client = Client()
-
-    # Test dataset
-    dataset_id = "e3101cae-af77-476f-a331-eb2e92e809e6"
-    dataset = DATASET
-    # Few shot dataset
-    dataset_id = "d833a66e-4074-413b-9c6d-313b15e307c8"
-    dataset = FEW_SHOT_DATASET
-
-    for example in dataset:
+    for example in examples:
         client.create_example(
-            inputs={
-                "question": example["question"],
-            },
-            outputs={
-                "reference": example["tool_calls"],
-            },
+            inputs={"question": example["question"]},
+            outputs={"reference": example["tool_calls"]},
             dataset_id=dataset_id,
         )
-

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langchain-benchmarks"
-version = "0.0.12"
+version = "0.0.13"
 description = "🦜💪 Flex those feathers!"
 authors = ["LangChain AI"]
 license = "MIT"

scripts/extract_task.py

@@ -1,253 +0,0 @@
-import json
-import sys
-sys.path.append("../langchain_benchmarks")
-from tool_usage.tasks.extraction_query import *
-from datetime import datetime
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_openai import ChatOpenAI
-from langsmith.evaluation import evaluate
-from langchain.chat_models import init_chat_model
-from langsmith.evaluation.evaluator import (
-    EvaluationResult,
-    EvaluationResults,
-)
-from langsmith.schemas import Example, Run
-from typing import Optional
-from langsmith.client import Client
-from langchain_core.messages import AIMessage, ToolMessage, HumanMessage
-from langchain_community.vectorstores import FAISS
-from langchain_core.example_selectors import SemanticSimilarityExampleSelector
-from langchain_openai import OpenAIEmbeddings
-from langchain_core.prompts.few_shot import FewShotChatMessagePromptTemplate
-from collections import Counter
-from langchain_core.utils.function_calling import convert_pydantic_to_openai_tool
-from vertexai.generative_models import FunctionDeclaration
-
-
-def calculate_recall(A, B):
-    # Count the occurrences of each element in A and B
-    count_A = Counter(A)
-    count_B = Counter(B)
-    
-    # Calculate the number of true positives
-    true_positives = sum(min(count_A[elem], count_B.get(elem, 0)) for elem in count_A)
-    
-    # Calculate recall
-    recall = true_positives / sum(count_A.values()) if count_A else 0
-    
-    return recall
-
-client = Client()
-
-def is_iso_format(date_str):
-    if not isinstance(date_str,str):
-        return False
-    try:
-        # Try to parse the string with datetime.fromisoformat
-        datetime.fromisoformat(date_str)
-        return True
-    except ValueError:
-        return False
-
-llm_judge = ChatOpenAI(model="gpt-4o")
-
-judge_prompt = ChatPromptTemplate.from_messages(
-    [
-        (
-            "system",
-            "You are an llm tasked with determining if the subject extracted by another LLM is an accurate "
-            "representation of the correct answer. You are to check for general semantic similarity since the words might not "
-            "match up perfectly but the meaning might still be the same. Return YES if the answers match, and NO otherwise. "
-            "Never return anything other than YES or NO.",
-        ),
-        (
-            "human",
-            "Is this query: {run_query} somewhat similar to this reference query: {reference_query}",
-        ),
-    ]
-)
-
-judge_chain = judge_prompt | llm_judge | StrOutputParser()
-
-tools = [DocQuery, TweetQuery, BlogQuery]
-
-def compare_outputs(run_outputs: dict, example_outputs: dict) -> EvaluationResults:
-    if len(run_outputs['response'].tool_calls) == 0:
-        correct_tool_score, determinstic_score, underministic_score = 0,0,0
-    else:
-        # Chose the correct tool
-        reference_tools = [tool["name"] for tool in example_outputs["reference"]]
-        outputted_tools = [tool["name"] for tool in run_outputs["response"].tool_calls]
-        correct_tool_score = calculate_recall(reference_tools,outputted_tools)
-
-        # Has the correct determenistic args
-        determinstic_score = 0
-        # Has the correct undetermenistic args
-        underministic_score = 0
-
-        if correct_tool_score == 1:
-            determinstic_score, underministic_score = 1, 1
-            for tool in example_outputs["reference"]:
-                corresponding_response_tool = [
-                        t
-                        for t in run_outputs["response"].tool_calls
-                        if t["name"] == tool["name"]
-                    ][0]["args"]
-                for arg in tool["args"]:
-                    if arg in ["query", "subject"]:
-                        ans = judge_chain.invoke(
-                            {
-                                "run_query": corresponding_response_tool[arg],
-                                "reference_query": tool["args"][arg],
-                            }
-                        )
-                        underministic_score = 1 if ans == "YES" else 0
-                    else:
-                        if (
-                            tool["args"][arg] and arg not in corresponding_response_tool
-                        ) or (
-                            tool["args"][arg]
-                            and not (tool["args"][arg] == corresponding_response_tool[arg])
-                            and not (is_iso_format(tool["args"][arg]) 
-                                and is_iso_format(corresponding_response_tool[arg]) 
-                                and datetime.fromisoformat(
-                                    (corresponding_response_tool[arg])
-                                ).replace(tzinfo=None)
-                                == datetime.fromisoformat(tool["args"][arg])
-                            )
-                        ):
-                            determinstic_score = 0
-    # Overall correctness
-    overall_score = int(
-        correct_tool_score == 1
-        and bool(determinstic_score)
-        and bool(underministic_score)
-    )
-    results = [
-        EvaluationResult(
-            key="Correct tool",
-            score=correct_tool_score,
-        ),
-        EvaluationResult(
-            key="Correct determenistic args",
-            score=determinstic_score,
-        ),
-        EvaluationResult(
-            key="Correct undermenistic args",
-            score=underministic_score,
-        ),
-        EvaluationResult(
-            key="Overall correctness",
-            score=overall_score,
-        ),
-    ]
-
-    return {"results": results}
-
-
-def evaluate_run(run: Run, example: Optional[Example] = None) -> EvaluationResults:
-    return compare_outputs(run.outputs, example.outputs)
-
-
-uncleaned_examples = [
-    e
-    for e in client.list_examples(
-        dataset_name="Extraction Task Few Shot"
-    )
-]
-static_indices = [0,2,5]
-few_shot_messages, few_shot_str = [], ""
-few_shot_messages_by_index = {}
-examples_for_semantic_search = []
-
-for j,example in enumerate(uncleaned_examples):
-    few_shot_messages_for_example = []
-    few_shot_messages_for_example.append(HumanMessage(name="example_human",content=example.inputs['question'][0]['content']))
-    few_shot_messages_for_example.append(AIMessage(name="example_assistant", content="", tool_calls=[{"name": tc["name"], "args": tc["args"], "type": "tool_call", "id": f"{10*j+i}"} for i, tc in enumerate(example.outputs["reference"])]))
-    few_shot_str += f"Human message: {example.inputs['question'][0]['content']}"
-    few_shot_str += "\n"
-    for i, tool_call in enumerate(example.outputs["reference"]):
-        few_shot_messages_for_example.append(ToolMessage("You have correctly called this tool", name=tool_call["name"], tool_call_id=f"{10*j+i}"))
-        few_shot_str += f"AI Message: Tool Call: Name: {tool_call['name']} Args: {{{', '.join(f'{k}: {v}' for k,v in tool_call['args'].items())}}}"
-        few_shot_str += "\n"
-
-    few_shot_messages += few_shot_messages_for_example
-    few_shot_messages_by_index[j] = few_shot_messages_for_example
-    examples_for_semantic_search.append({"question":example.inputs['question'][0]['content'],"messages":few_shot_messages_for_example})
-
-
-prompt = ChatPromptTemplate.from_messages(
-                    [
-                        ("system", "{instructions}"),
-                        MessagesPlaceholder("few_shot_message_list"),
-                        ("human", "{input}"),
-                    ]
-)
-
-
-
-def predict_for_model(model,instructions,few_shot_method,model_name):
-    few_shot_message_list = []
-    if "groq" not in model_name:
-        chain = prompt | model.bind_tools(tools,tool_choice="any").with_retry(stop_after_attempt=5)
-    else:
-        chain = prompt | model.bind_tools(tools).with_retry(stop_after_attempt=5)
-    if few_shot_method == "few-shot-string":
-        instructions += f"\n Here are some examples: \n {few_shot_str}"
-    elif few_shot_method == "few-shot-messages":
-        few_shot_message_list = few_shot_messages
-    elif few_shot_method == "few-shot-static-messages":
-        few_shot_message_list = [message for index in static_indices for message in few_shot_messages_by_index[index]]
-    elif few_shot_method == "few-shot-dynamic-messages":
-        def predict(example: dict):
-            example_selector = SemanticSimilarityExampleSelector.from_examples(
-                examples_for_semantic_search,
-                OpenAIEmbeddings(),
-                FAISS,
-                k=3,
-                input_keys=["question"],
-                example_keys=["messages"],
-            )
-
-            few_shot_prompt = FewShotChatMessagePromptTemplate(
-                input_variables=[],
-                example_selector=example_selector,
-                example_prompt=MessagesPlaceholder("messages"),
-            )
-            return {"response": chain.invoke({"input":example["question"],"instructions":instructions,"few_shot_message_list":few_shot_prompt.invoke({"question":example["question"][0]['content']}).messages})}
-        return predict
-    
-    def predict(example: dict):
-        return {"response": chain.invoke({"input":example["question"],"instructions":instructions,"few_shot_message_list":few_shot_message_list})}
-    
-    
-    return predict
-
-
-models = [("claude-3-haiku-20240307","anthropic",),
-            ("claude-3-sonnet-20240229","anthropic",),
-            ("claude-3-opus-20240229","anthropic",),
-            ("claude-3-5-sonnet-20240620","anthropic",),
-          ("gpt-3.5-turbo-0125","openai"),
-          ("gpt-4o","openai"),
-          ("gpt-4o-mini","openai"),
-          ("gemini-1.5-pro","google_vertexai"),
-    ("gemini-1.5-flash","google_vertexai"),
-            ("llama3-groq-70b-8192-tool-use-preview","groq"),
-            ("llama3-groq-8b-8192-tool-use-preview","groq"),]
-
-few_shot_methods = ["no-few-shot","few-shot-string","few-shot-messages","few-shot-static-messages","few-shot-dynamic-messages"]
-
-from tqdm import tqdm
-for i in tqdm(range(2)):
-    for model_name, model_provider in models[-4:-2]:
-        model = init_chat_model(model_name,model_provider=model_provider)
-        for few_shot_method in few_shot_methods[2:]:
-            evaluate(
-                predict_for_model(model,EXTRACTION_TASK.instructions,few_shot_method,model_name),
-                data=EXTRACTION_TASK.name,
-                evaluators=[evaluate_run],
-                experiment_prefix=f"{model_name}-TEST-{i+2}-{few_shot_method}",
-            )
-            

scripts/multiverse_math_benchmark.py

@@ -0,0 +1,192 @@
+import datetime
+import sys
+import uuid
+
+from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
+from langchain_core.messages.utils import convert_to_messages
+from langsmith.client import Client
+
+from langchain_benchmarks import __version__
+
+sys.path.append("./../langchain_benchmarks")
+from langchain.agents import AgentExecutor, create_tool_calling_agent
+from langchain.chat_models import init_chat_model
+from langsmith.evaluation import evaluate
+from tool_usage.tasks.multiverse_math import *
+
+tests = [
+    (
+        "claude-3-haiku-20240307",
+        "anthropic",
+    ),
+    (
+        "claude-3-sonnet-20240229",
+        "anthropic",
+    ),
+    (
+        "claude-3-opus-20240229",
+        "anthropic",
+    ),
+    (
+        "claude-3-5-sonnet-20240620",
+        "anthropic",
+    ),
+    ("gpt-3.5-turbo-0125", "openai"),
+    (
+        "gpt-4o",
+        "openai",
+    ),
+    ("gpt-4o-mini", "openai"),
+]
+
+client = Client()  # Launch langsmith client for cloning datasets
+
+
+def get_few_shot_messages(task_name):
+    if task_name == "Multiverse Math":
+        uncleaned_examples = [
+            e
+            for e in client.list_examples(
+                dataset_name="multiverse-math-examples-for-few-shot"
+            )
+        ]
+        few_shot_messages = []
+        few_shot_three_messages = []
+        examples = []
+        for i in range(len(uncleaned_examples)):
+            converted_messages = convert_to_messages(
+                uncleaned_examples[i].outputs["output"]
+            )
+            examples.append(
+                # The message at index 1 is the human message asking the actual math question (0th message is system prompt)
+                {
+                    "question": converted_messages[1].content,
+                    "messages": [
+                        m
+                        for m in converted_messages
+                        if isinstance(m, SystemMessage) == False
+                    ],
+                }
+            )
+            few_shot_messages += converted_messages
+            if i < 3:
+                few_shot_three_messages += converted_messages
+
+        return (
+            examples,
+            [m for m in few_shot_messages if not isinstance(m, SystemMessage)],
+            [m for m in few_shot_three_messages if not isinstance(m, SystemMessage)],
+        )
+    else:
+        raise ValueError("Few shot messages not supported for this dataset")
+
+
+def turn_messages_to_str(few_shot_messages):
+    few_shot_str = ""
+    for m in few_shot_messages:
+        if isinstance(m.content, list):
+            few_shot_str += "<|im_start|>assistant"
+            for tool_use in m.content:
+                if "name" in tool_use:
+                    few_shot_str += f"Use tool {tool_use['name']}, input: {', '.join(f'{k}:{v}' for k,v in tool_use['input'].items())}"
+                else:
+                    few_shot_str += tool_use["text"]
+                few_shot_str += "\n"
+            few_shot_str += "\n<|im_end|>"
+        else:
+            if isinstance(m, HumanMessage):
+                few_shot_str += f"<|im_start|>user\n{m.content}\n<|im_end|>"
+            elif isinstance(m, ToolMessage):
+                few_shot_str += f"<|im_start|>tool\n{m.content}\n<|im_end|>"
+            else:
+                few_shot_str += f"<|im_start|>assistant\n{m.content}\n<|im_end|>"
+
+        few_shot_str += "\n"
+    return few_shot_str
+
+
+def get_few_shot_str_from_messages(few_shot_messages, few_shot_three_messages):
+    few_shot_str = turn_messages_to_str(few_shot_messages)
+    few_shot_three_str = turn_messages_to_str(few_shot_three_messages)
+    return few_shot_str, few_shot_three_str
+
+
+def get_prompts(task_name, **kwargs):
+    if task_name == "Multiverse Math":
+        return [
+            (
+                client.pull_prompt("langchain-ai/multiverse-math-no-few-shot"),
+                "no-few-shot",
+            ),
+            (
+                client.pull_prompt("langchain-ai/multiverse-math-few-shot-messages"),
+                "few-shot-messages",
+            ),
+            (
+                client.pull_prompt("langchain-ai/multiverse-math-few-shot-str"),
+                "few-shot-string",
+            ),
+            (
+                client.pull_prompt("langchain-ai/multiverse-math-few-shot-3-messages"),
+                "few-shot-three-messages",
+            ),
+            (
+                client.pull_prompt("langchain-ai/multiverse-math-few-shot-3-str"),
+                "few-shot-three-strings",
+            ),
+        ]
+
+
+def predict_from_callable(callable, instructions):
+    def predict(run):
+        return callable.invoke(
+            {"question": run["question"], "instructions": instructions}
+        )
+
+    return predict
+
+
+experiment_uuid = uuid.uuid4().hex[:4]
+today = datetime.date.today().isoformat()
+
+task = MULTIVERSE_MATH
+dataset_name = task.name
+examples, few_shot_messages, few_shot_three_messages = get_few_shot_messages(task.name)
+few_shot_str, few_shot_three_str = get_few_shot_str_from_messages(
+    few_shot_messages, few_shot_three_messages
+)
+
+prompts = get_prompts(
+    task.name,
+    examples=examples,
+    few_shot_three_messages=few_shot_three_messages,
+    few_shot_three_str=few_shot_three_str,
+)
+
+for model_name, model_provider in tests:
+    model = init_chat_model(model_name, model_provider=model_provider, temperature=0)
+
+    print(f"Benchmarking {task.name} with model: {model_name}")
+    eval_config = task.get_eval_config()
+
+    for prompt, prompt_name in prompts:
+        tools = task.create_environment().tools
+        agent = create_tool_calling_agent(model, tools, prompt)
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, return_intermediate_steps=True
+        )
+
+        evaluate(
+            predict_from_callable(agent_executor, task.instructions),
+            data=dataset_name,
+            evaluators=eval_config.custom_evaluators,
+            max_concurrency=5,
+            metadata={
+                "model": model_name,
+                "id": experiment_uuid,
+                "task": task.name,
+                "date": today,
+                "langchain_benchmarks_version": __version__,
+            },
+            experiment_prefix=f"{model_name}-{task.name}-{prompt_name}",
+        )

scripts/query_analysis_benchmark.py

@@ -0,0 +1,331 @@
+import uuid
+from collections import Counter
+from datetime import datetime
+from typing import Optional
+
+from langchain.chat_models import init_chat_model
+from langchain_community.vectorstores import FAISS
+from langchain_core.example_selectors import SemanticSimilarityExampleSelector
+from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import (
+    ChatPromptTemplate,
+    FewShotChatMessagePromptTemplate,
+    MessagesPlaceholder,
+)
+from langchain_openai import OpenAIEmbeddings
+from langsmith.client import Client
+from langsmith.evaluation import evaluate
+from langsmith.evaluation.evaluator import EvaluationResult, EvaluationResults
+from langsmith.schemas import Example, Run
+
+from langchain_benchmarks.tool_usage.tasks.query_analysis import (
+    QUERY_ANALYSIS_TASK,
+    BlogQuery,
+    DocQuery,
+    TweetQuery,
+)
+
+
+def calculate_recall(A, B):
+    # Count the occurrences of each element in A and B
+    count_A = Counter(A)
+    count_B = Counter(B)
+
+    # Calculate the number of true positives
+    true_positives = sum(min(count_A[elem], count_B.get(elem, 0)) for elem in count_A)
+
+    # Calculate recall
+    recall = true_positives / sum(count_A.values()) if count_A else 0
+
+    return recall
+
+
+client = Client()
+
+
+def is_iso_format(date_str):
+    if not isinstance(date_str, str):
+        return False
+    try:
+        # Try to parse the string with datetime.fromisoformat
+        datetime.fromisoformat(date_str)
+        return True
+    except ValueError:
+        return False
+
+
+llm_judge = init_chat_model("gpt-4o")
+
+judge_prompt = ChatPromptTemplate.from_messages(
+    [
+        (
+            "system",
+            "You are an llm tasked with determining if the subject extracted by another LLM is an accurate "
+            "representation of the correct answer. You are to check for general semantic similarity since the words might not "
+            "match up perfectly but the meaning might still be the same. Return YES if the answers match, and NO otherwise. "
+            "Never return anything other than YES or NO.",
+        ),
+        (
+            "human",
+            "Is this query: {run_query} somewhat similar to this reference query: {reference_query}",
+        ),
+    ]
+)
+
+judge_chain = judge_prompt | llm_judge | StrOutputParser()
+
+tools = [DocQuery, TweetQuery, BlogQuery]
+
+
+def compare_outputs(run_outputs: dict, example_outputs: dict) -> EvaluationResults:
+    if len(run_outputs["response"].tool_calls) == 0:
+        correct_tool_score, deterministic_score, nondeterministic_score = 0, 0, 0
+    else:
+        # Chose the correct tool
+        reference_tools = [tool["name"] for tool in example_outputs["reference"]]
+        outputted_tools = [tool["name"] for tool in run_outputs["response"].tool_calls]
+        correct_tool_score = calculate_recall(reference_tools, outputted_tools)
+
+        # Has the correct deterministic args
+        deterministic_score = 0
+        # Has the correct in-deterministic args
+        nondeterministic_score = 0
+
+        if correct_tool_score == 1:
+            deterministic_score, nondeterministic_score = 1, 1
+            for tool in example_outputs["reference"]:
+                corresponding_response_tool = [
+                    t
+                    for t in run_outputs["response"].tool_calls
+                    if t["name"] == tool["name"]
+                ][0]["args"]
+                for arg in tool["args"]:
+                    if arg in ["query", "subject"]:
+                        ans = judge_chain.invoke(
+                            {
+                                "run_query": corresponding_response_tool[arg],
+                                "reference_query": tool["args"][arg],
+                            }
+                        )
+                        nondeterministic_score = 1 if ans == "YES" else 0
+                    else:
+                        if (
+                            tool["args"][arg] and arg not in corresponding_response_tool
+                        ) or (
+                            tool["args"][arg]
+                            and not (
+                                tool["args"][arg] == corresponding_response_tool[arg]
+                            )
+                            and not (
+                                is_iso_format(tool["args"][arg])
+                                and is_iso_format(corresponding_response_tool[arg])
+                                and datetime.fromisoformat(
+                                    (corresponding_response_tool[arg])
+                                ).replace(tzinfo=None)
+                                == datetime.fromisoformat(tool["args"][arg])
+                            )
+                        ):
+                            deterministic_score = 0
+    # Overall correctness
+    overall_score = int(
+        correct_tool_score == 1
+        and bool(deterministic_score)
+        and bool(nondeterministic_score)
+    )
+    results = [
+        EvaluationResult(
+            key="Correct tool",
+            score=correct_tool_score,
+        ),
+        EvaluationResult(
+            key="Correct deterministic args",
+            score=deterministic_score,
+        ),
+        EvaluationResult(
+            key="Correct nondeterministic args",
+            score=nondeterministic_score,
+        ),
+        EvaluationResult(
+            key="Overall correctness",
+            score=overall_score,
+        ),
+    ]
+
+    return {"results": results}
+
+
+def evaluate_run(run: Run, example: Optional[Example] = None) -> EvaluationResults:
+    return compare_outputs(run.outputs, example.outputs)
+
+
+uncleaned_examples = [
+    e for e in client.list_examples(dataset_name="Extraction Task Few Shot")
+]
+static_indices = [0, 2, 5]
+few_shot_messages, few_shot_str = [], ""
+few_shot_messages_by_index = {}
+examples_for_semantic_search = []
+
+for j, example in enumerate(uncleaned_examples):
+    few_shot_messages_for_example = []
+    few_shot_messages_for_example.append(
+        HumanMessage(
+            name="example_human", content=example.inputs["question"][0]["content"]
+        )
+    )
+    few_shot_messages_for_example.append(
+        AIMessage(
+            name="example_assistant",
+            content="",
+            tool_calls=[
+                {
+                    "name": tc["name"],
+                    "args": tc["args"],
+                    "type": "tool_call",
+                    "id": f"{10*j+i}",
+                }
+                for i, tc in enumerate(example.outputs["reference"])
+            ],
+        )
+    )
+    few_shot_str += (
+        f"<|im_start|>user\n{example.inputs['question'][0]['content']}\n<|im_end|>"
+    )
+    few_shot_str += "\n<|im_start|>assistant\n"
+    for i, tool_call in enumerate(example.outputs["reference"]):
+        few_shot_messages_for_example.append(
+            ToolMessage(
+                "You have correctly called this tool",
+                name=tool_call["name"],
+                tool_call_id=f"{10*j+i}",
+            )
+        )
+        few_shot_str += f"Tool Call: Name: {tool_call['name']} Args: {{{', '.join(f'{k}: {v}' for k,v in tool_call['args'].items())}}}"
+        few_shot_str += "\n"
+    few_shot_str += "<|im_end|>"
+
+    few_shot_messages += few_shot_messages_for_example
+    few_shot_messages_by_index[j] = few_shot_messages_for_example
+    examples_for_semantic_search.append(
+        {
+            "question": example.inputs["question"][0]["content"],
+            "messages": few_shot_messages_for_example,
+        }
+    )
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "{instructions}"),
+        MessagesPlaceholder("few_shot_message_list"),
+        ("human", "{input}"),
+    ]
+)
+
+
+def predict_for_model(model, instructions, few_shot_method, model_name):
+    few_shot_message_list = []
+    chain = prompt | model.bind_tools(tools).with_retry(stop_after_attempt=5)
+    if few_shot_method == "few-shot-string":
+        instructions += f"\n Here are some examples: \n {few_shot_str}"
+    elif few_shot_method == "few-shot-messages":
+        few_shot_message_list = few_shot_messages
+    elif few_shot_method == "few-shot-static-messages":
+        few_shot_message_list = [
+            message
+            for index in static_indices
+            for message in few_shot_messages_by_index[index]
+        ]
+    elif few_shot_method == "few-shot-dynamic-messages":
+
+        def predict(example: dict):
+            example_selector = SemanticSimilarityExampleSelector.from_examples(
+                examples_for_semantic_search,
+                OpenAIEmbeddings(model="text-embedding-3-large"),
+                FAISS,
+                k=3,
+                input_keys=["question"],
+                example_keys=["messages"],
+            )
+
+            few_shot_prompt = FewShotChatMessagePromptTemplate(
+                input_variables=[],
+                example_selector=example_selector,
+                example_prompt=MessagesPlaceholder("messages"),
+            )
+            return {
+                "response": chain.invoke(
+                    {
+                        "input": example["question"],
+                        "instructions": instructions,
+                        "few_shot_message_list": few_shot_prompt.invoke(
+                            {"question": example["question"][0]["content"]}
+                        ).messages,
+                    }
+                )
+            }
+
+        return predict
+
+    def predict(example: dict):
+        return {
+            "response": chain.invoke(
+                {
+                    "input": example["question"],
+                    "instructions": instructions,
+                    "few_shot_message_list": few_shot_message_list,
+                }
+            )
+        }
+
+    return predict
+
+
+models = [
+    (
+        "claude-3-haiku-20240307",
+        "anthropic",
+    ),
+    (
+        "claude-3-sonnet-20240229",
+        "anthropic",
+    ),
+    (
+        "claude-3-opus-20240229",
+        "anthropic",
+    ),
+    (
+        "claude-3-5-sonnet-20240620",
+        "anthropic",
+    ),
+    ("gpt-3.5-turbo-0125", "openai"),
+    ("gpt-4o", "openai"),
+    ("gpt-4o-mini", "openai"),
+]
+
+few_shot_methods = [
+    "no-few-shot",
+    "few-shot-string",
+    "few-shot-messages",
+    "few-shot-static-messages",
+    "few-shot-dynamic-messages",
+]
+
+from tqdm import tqdm
+
+experiment_uuid = uuid.uuid4().hex[:4]
+for i in tqdm(range(3)):
+    for model_name, model_provider in models:
+        model = init_chat_model(
+            model_name, model_provider=model_provider, temperature=0
+        )
+        for few_shot_method in few_shot_methods:
+            evaluate(
+                predict_for_model(
+                    model, QUERY_ANALYSIS_TASK.instructions, few_shot_method, model_name
+                ),
+                data=QUERY_ANALYSIS_TASK.name,
+                evaluators=[evaluate_run],
+                experiment_prefix=f"{model_name}-TEST-{i+2}-{few_shot_method}",
+                metadata={"id": experiment_uuid},
+            )