Eval for extraction

extraction/eval_extraction.ipynb

@@ -0,0 +1,224 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ec706820",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langsmith import Client\n",
+    "from langchain.smith import RunEvalConfig, run_on_dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c41de852",
+   "metadata": {},
+   "source": [
+    "Methodology -\n",
+    "\n",
+    "1. Create `dataset` in LangSmith w/ curated extraction examples, `Extraction Benchmark Dataset`\n",
+    "2. Eval w/ various extraction approaches\n",
+    "\n",
+    "* OpenAI function: `get_oai_function_chain`\n",
+    "* OpenAI w/o function: `get_oai_chain`\n",
+    "* [Anthropic w/ function](https://python.langchain.com/docs/integrations/chat/anthropic_functions): `get_anthropic_function_chain`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "5092d734",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Knowledge triplet schema\n",
+    "extraction_schema = {\n",
+    "    \"properties\": {\n",
+    "        \"subject\": {\"type\": \"string\"},\n",
+    "        \"predicate\": {\"type\": \"string\"},\n",
+    "        \"object\": {\"type\": \"string\"},\n",
+    "    },\n",
+    "    \"required\": [\"subject\", \"predicate\", \"object\"],\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "ba63af94",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.chains import create_extraction_chain\n",
+    "\n",
+    "def get_oai_function_chain():\n",
+    "    \n",
+    "    # Chain w/ function call\n",
+    "    llm = ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo\")\n",
+    "    chain = create_extraction_chain(extraction_schema, llm)\n",
+    "    return chain"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "6fe0c2cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain import PromptTemplate\n",
+    "from langchain import PromptTemplate, LLMChain\n",
+    "\n",
+    "def get_oai_chain():\n",
+    "    \n",
+    "    # Prompt \n",
+    "    EXTRACTION_PROMPT = PromptTemplate(\n",
+    "        input_variables=[\"input_text\"],\n",
+    "        template=\"\"\"Given the following input text, perform extraction of knowledge graph triples as a JSON with/\n",
+    "        \"subject\":,\"predicate\":,\"object\":. Here is the input text: {input_text}\"\"\",\n",
+    "    )\n",
+    "    # LLM chain\n",
+    "    llm = ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo\")\n",
+    "    llm_chain = LLMChain(prompt=EXTRACTION_PROMPT,llm=llm)\n",
+    "    return llm_chain"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "904fc05d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chains import create_extraction_chain\n",
+    "from langchain_experimental.llms.anthropic_functions import AnthropicFunctions\n",
+    "\n",
+    "def get_anthropic_function_chain():\n",
+    "    \n",
+    "    # Chain w/ function call\n",
+    "    model = AnthropicFunctions(model='claude-2')\n",
+    "    chain = create_extraction_chain(extraction_schema, model)\n",
+    "    return chain"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1ef8fe8",
+   "metadata": {},
+   "source": [
+    "Eval"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "97ca5851",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "View the evaluation results for project 'f7683fbad79f4677b54950283dda1f32-LLMChain' at:\n",
+      "https://langchain.plus/projects/p/b4fc624d-29c3-4f27-8c4b-9e446adc367c?eval=true\n"
+     ]
+    }
+   ],
+   "source": [
+    "client = Client()\n",
+    "\n",
+    "eval_config = RunEvalConfig(\n",
+    "    evaluators=[\n",
+    "        \"qa\"\n",
+    "    ],\n",
+    ")\n",
+    "\n",
+    "chain_results = run_on_dataset(\n",
+    "    client,\n",
+    "    dataset_name=\"Extraction Benchmark Dataset\",\n",
+    "    llm_or_chain_factory=get_oai_function_chain,\n",
+    "    evaluation=eval_config,\n",
+    ")    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "64e8e510",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "View the evaluation results for project '1497281881d4429c8f87de5e19a26f7f-LLMChain' at:\n",
+      "https://langchain.plus/projects/p/bc1860f9-47a8-4235-abc4-7628e5d993d7?eval=true\n"
+     ]
+    }
+   ],
+   "source": [
+    "client = Client()\n",
+    "\n",
+    "eval_config = RunEvalConfig(\n",
+    "    evaluators=[\n",
+    "        \"qa\"\n",
+    "    ],\n",
+    ")\n",
+    "\n",
+    "chain_results = run_on_dataset(\n",
+    "    client,\n",
+    "    dataset_name=\"Extraction Benchmark Dataset\",\n",
+    "    llm_or_chain_factory=get_oai_chain,\n",
+    "    evaluation=eval_config,\n",
+    ")    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8c995c6f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = Client()\n",
+    "\n",
+    "eval_config = RunEvalConfig(\n",
+    "    evaluators=[\n",
+    "        \"qa\"\n",
+    "    ],\n",
+    ")\n",
+    "\n",
+    "chain_results = run_on_dataset(\n",
+    "    client,\n",
+    "    dataset_name=\"Extraction Benchmark Dataset\",\n",
+    "    llm_or_chain_factory=get_anthropic_function_chain,\n",
+    "    evaluation=eval_config,\n",
+    ")    "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}