mirror of
https://github.com/langchain-ai/pinecone-serverless.git
synced 2026-07-01 10:04:31 -04:00
Lance Martin
193 lines
7.1 KiB
Plaintext
193 lines
7.1 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "9e545fe7-e7ec-4b16-89ae-6820bb534454",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"import os\n",
|
||
"from langchain_community.chat_models import ChatOpenAI\n",
|
||
"from langchain_community.embeddings import CohereEmbeddings\n",
|
||
"from langchain_community.vectorstores import Pinecone\n",
|
||
"from langchain_core.output_parsers import StrOutputParser\n",
|
||
"from langchain_core.prompts import ChatPromptTemplate\n",
|
||
"from langchain_core.runnables import RunnableParallel, RunnablePassthrough, RunnableLambda\n",
|
||
"from pinecone import Pinecone as PineconeClient\n",
|
||
"import requests"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 8,
|
||
"id": "6eed122c-aa7e-4856-ba01-66d6362d9758",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Keys\n",
|
||
"PINECONE_API_KEY = os.environ[\"PINECONE_API_KEY\"]\n",
|
||
"PINECONE_ENVIRONMENT = os.environ[\"PINECONE_ENVIRONMENT\"]\n",
|
||
"PINECONE_INDEX_NAME = os.environ[\"PINECONE_INDEX_NAME\"]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "5621b9f9-f277-4d06-8bb0-72644792f2f8",
|
||
"metadata": {},
|
||
"source": [
|
||
"Serverless index from [this dataset](https://huggingface.co/datasets/Cohere/wikipedia-22-12)."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"id": "3de2de5e-aad0-4ca3-b816-f2cb07b2d251",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Init\n",
|
||
"pinecone = PineconeClient(api_key=PINECONE_API_KEY,\n",
|
||
" environment=PINECONE_ENVIRONMENT)\n",
|
||
"\n",
|
||
"embeddings = CohereEmbeddings(model=\"multilingual-22-12\")\n",
|
||
"vectorstore = Pinecone.from_existing_index(index_name=PINECONE_INDEX_NAME, embedding=embeddings)\n",
|
||
"retriever = vectorstore.as_retriever()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"id": "9586b0fa-8555-4bcc-8cfc-ffea62fddc67",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# RAG prompt\n",
|
||
"template = \"\"\"Answer the question based only on the following context:\n",
|
||
"{context}\n",
|
||
"Question: {question}\n",
|
||
"\"\"\"\n",
|
||
"prompt = ChatPromptTemplate.from_template(template)\n",
|
||
"\n",
|
||
"# RAG\n",
|
||
"model = ChatOpenAI(temperature=0, \n",
|
||
" model=\"gpt-4-1106-preview\")\n",
|
||
"\n",
|
||
"chain = (\n",
|
||
" RunnableParallel({\"context\": retriever, \"question\": RunnablePassthrough()})\n",
|
||
" | prompt\n",
|
||
" | model\n",
|
||
" | StrOutputParser()\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 11,
|
||
"id": "81c13ad6-cebb-4789-a9e6-816bca4e66bb",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"'Film noir is a genre of film that is characterized by a set of attributes that provoke ongoing debate regarding its definition. According to French critics, including Étienne Chaumeton, in their 1955 book \"Panorama du film noir américain 1941–1953\" (\"A Panorama of American Film Noir\"), film noir can be described as oneiric (dreamlike), strange, erotic, ambivalent, and cruel, although not every film noir necessarily embodies all these attributes to the same degree. The genre is known for its complexity, with films that may feature crime and violence, complex characters and plot-lines, mystery, and moral ambivalence.\\n\\nFilm noir is also recognized for its visual style, which may include the use of chiaroscuro lighting techniques, and is often associated with a bleak societal perspective, offering a critique on global capitalism and consumerism. This is particularly evident in the neon-noir sub-genre, which emphasizes the socio-critical aspects of film noir and often includes long shots or montages of dark and menacing cityscapes.\\n\\nThe defining characteristics of film noir are a source of controversy among critics, with some focusing on the genre\\'s tragic or bleak conclusions, distinctive visual style, plot and character types, mood, and attitude. Despite the many attempts to define film noir, it remains an elusive phenomenon that is difficult to pin down with a definitive set of identifying characteristics.'"
|
||
]
|
||
},
|
||
"execution_count": 11,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"chain.invoke(\"what is film noir?\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "4568a49d-91f7-4d5f-8514-9f5a447f5027",
|
||
"metadata": {},
|
||
"source": [
|
||
"Extract full wiki page."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 12,
|
||
"id": "ea74ba28-668c-4bc0-b262-568298e13533",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def fetch_wikipedia_page(id):\n",
|
||
" url = f\"https://en.wikipedia.org/w/api.php?action=query&prop=extracts&format=json&pageids={id}\"\n",
|
||
" response = requests.get(url)\n",
|
||
" data = response.json()\n",
|
||
" page_content = list(data['query']['pages'].values())[0]['extract']\n",
|
||
" return page_content\n",
|
||
"\n",
|
||
"def fetch_url(x):\n",
|
||
" urls = [doc.metadata['url'] for doc in x['context']]\n",
|
||
" ids = [url.split('=')[-1] for url in urls]\n",
|
||
" # First 32k tokens\n",
|
||
" contents = [fetch_wikipedia_page(id)[:32000] for id in ids] \n",
|
||
" return {\"context\": contents, \"question\": x[\"question\"]}\n",
|
||
"\n",
|
||
"# RAG\n",
|
||
"model = ChatOpenAI(temperature=0, \n",
|
||
" model=\"gpt-4-1106-preview\")\n",
|
||
"\n",
|
||
"chain = (\n",
|
||
" RunnableParallel({\"context\": retriever, \"question\": RunnablePassthrough()})\n",
|
||
" | RunnableLambda(fetch_url) \n",
|
||
" | prompt\n",
|
||
" | model\n",
|
||
" | StrOutputParser()\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 13,
|
||
"id": "10d7eb82-c9ad-4ddd-a543-72b7fa40d009",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"'Film noir is a cinematic term used primarily to describe stylized Hollywood crime dramas, particularly those that emphasize cynical attitudes and motivations. The classic period of American film noir is generally regarded as the 1940s and 1950s. Film noir of this era is associated with a low-key, black-and-white visual style that has roots in German Expressionist cinematography. It encompasses a range of plots and central figures, including private investigators, plainclothes police officers, aging boxers, hapless grifters, law-abiding citizens lured into a life of crime, femme fatales, or victims of circumstance.'"
|
||
]
|
||
},
|
||
"execution_count": 13,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"chain.invoke(\"what is film noir?\")"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.11.4"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|