mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 12:05:23 -04:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Pierre-Loic Doulcet
|
2e6c064682 | ||
|
Jerry Liu
|
4252f6186b | ||
|
Jerry Liu
|
22148ade9f | ||
|
Jerry Liu
|
b8332fe8e1 | ||
|
Ravi Theja
|
e40e92a133 |
@@ -4,9 +4,11 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Llama Parser <> LlamaIndex\n",
|
||||
"# Advanced RAG with LlamaParse\n",
|
||||
"\n",
|
||||
"This notebook is a complete walkthrough for using `LlamaParse` for RAG applications with `LlamaIndex`.\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_advanced.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook shows you how to use LlamaParse with our advanced markdown ingestion and recursive retrieval algorithms to model tables/text within a document hierarchically. This lets you ask questions over both tables and text.\n",
|
||||
"\n",
|
||||
"Note for this example, we are using the `llama_index >=0.10.4` version"
|
||||
]
|
||||
|
||||
+91
-267
@@ -20,17 +20,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"zsh:1: command not found: pip\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install llama-index llama-parse"
|
||||
]
|
||||
@@ -46,29 +38,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"--2024-03-19 14:30:27-- https://policyholder.gov.in/documents/37343/931203/NBHTGBP22011V012223.pdf/c392bcc1-f6a8-cadd-ab84-495b3273d2c3?version=1.0&t=1669350459879&download=true\n",
|
||||
"Resolving policyholder.gov.in (policyholder.gov.in)... 13.107.246.61, 13.107.213.61\n",
|
||||
"Connecting to policyholder.gov.in (policyholder.gov.in)|13.107.246.61|:443... connected.\n",
|
||||
"HTTP request sent, awaiting response... 200 \n",
|
||||
"Length: 1341586 (1.3M) [application/pdf]\n",
|
||||
"Saving to: ‘./policy.pdf’\n",
|
||||
"\n",
|
||||
"./policy.pdf 100%[===================>] 1.28M 1.37MB/s in 0.9s \n",
|
||||
"\n",
|
||||
"2024-03-19 14:30:29 (1.37 MB/s) - ‘./policy.pdf’ saved [1341586/1341586]\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!wget \"https://policyholder.gov.in/documents/37343/931203/NBHTGBP22011V012223.pdf/c392bcc1-f6a8-cadd-ab84-495b3273d2c3?version=1.0&t=1669350459879&download=true\" -O \"./policy.pdf\""
|
||||
]
|
||||
@@ -82,7 +56,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -95,7 +69,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -106,7 +80,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -133,7 +107,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -142,7 +116,8 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Started parsing the file under job_id f6ef66a3-a085-4fa5-8300-479adcdea779\n"
|
||||
"Started parsing the file under job_id b8946573-c911-4e00-8921-1bad1cda3d64\n",
|
||||
"......"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -154,7 +129,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -193,7 +168,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -206,42 +181,18 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/Users/pierre/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
|
||||
" warnings.warn(\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Embeddings have been explicitly disabled. Using MockEmbedding.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"111it [00:00, 86184.33it/s]\n",
|
||||
"100%|██████████| 111/111 [00:35<00:00, 3.15it/s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"nodes = node_parser.get_nodes_from_documents(documents)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -254,7 +205,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -301,7 +252,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -310,7 +261,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 5b110ac2-e403-4eb3-a92b-97438d97199a\n",
|
||||
"Started parsing the file under job_id ec9e77c9-6ad9-4c9b-9efb-c9f659b0d481\n",
|
||||
"....."
|
||||
]
|
||||
}
|
||||
@@ -320,7 +271,7 @@
|
||||
"This document is an insurance policy.\n",
|
||||
"When a benefits/coverage/exlusion is describe in the document ammend to it add a text in the follwing benefits string format (where coverage could be an exclusion).\n",
|
||||
"\n",
|
||||
"Benefits for {nameofrisk} is {benefitsDescription}, with amount: {benefitsAmount}, and conditions: {benefitsCondition}. \n",
|
||||
"For {nameofrisk} and in this condition {whenDoesThecoverageApply} the coverage is {coverageDescription}. \n",
|
||||
" \n",
|
||||
"If the document contain a benefits TABLE that describe coverage amounts, do not ouput it as a table, but instead as a list of benefits string.\n",
|
||||
" \n",
|
||||
@@ -336,7 +287,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -345,149 +296,64 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"## Repatriation of Mortal remains\n",
|
||||
"## Inpatient treatment\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|5K, 10K, 15K, 20K, 25K, 50K, 75K, 100K, 150K, 200K 250K|Nil/50/100|\n",
|
||||
"Claim Form (filled and signed by pe Insured)\n",
|
||||
"Hospital Daily Cash\n",
|
||||
"Release of Medical information Form (filled and signed by pe Insured)\n",
|
||||
"Waiver of Deductible\n",
|
||||
"Original papological and diagnostic reports, discharge summary indoor case papers (if any) and prescriptions issued by pe treating Medical practitioner or Network Provider\n",
|
||||
"Optional Co-payment\n",
|
||||
"Adventure Sports Cover\n",
|
||||
"Home to Home Cover\n",
|
||||
"Passport and Visa copy wip Entry Stamp of Country of Visit and exit Stamp from India\n",
|
||||
"Extension to in-patient care\n",
|
||||
"Ambulance Charge\n",
|
||||
"FIR report of police (if applicable)\n",
|
||||
"\n",
|
||||
"## Repatriation of Mortal remains\n",
|
||||
"## Out-patient treatment\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|3Lac, 5Lac|Independent SI|\n",
|
||||
"Cancer Screening & Mammographic Examination\n",
|
||||
"Original bills and receipts for:\n",
|
||||
"1. Charges paid towards Hospital accommodation, nursing facilities, and oper medical services rendered\n",
|
||||
"2. Fees paid to pe Medical Practitioner and for special nursing charges\n",
|
||||
"3. Charges incurred towards any and all test and / or examinations rendered in connection wip pe treatment\n",
|
||||
"4. Charges incurred towards medicines or drugs purchased from a registered pharmacy oper pan pe Network provider duly supported by pe prescriptions of pe Medical Practitioner attending to pe Insured Person\n",
|
||||
"5. Any oper document as required by pe Company to assist pe Claim\n",
|
||||
"\n",
|
||||
"## Total Loss of Checked-in Baggage\n",
|
||||
"## Medical evacuation\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100 to 1000 in multiples of 100|NIL|\n",
|
||||
"Medical reports and transportation details issued by the evacuation agency, prescriptions and medical report by the attending Medical Practitioner furnishing the name of the Insured Person and details of treatment rendered along with the statement confirming the necessity of evacuation.\n",
|
||||
"\n",
|
||||
"## Delay of Checked-in Baggage\n",
|
||||
"Documentary proof for expenses incurred towards the Medical Evacuation.\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100 to 500 in multiples of 100|1/2/3/5/6/24 hours|\n",
|
||||
"|25 to 100 in multiple of 25|3/6/8/12 hours|\n",
|
||||
"## Compassionate visit\n",
|
||||
"\n",
|
||||
"## Trip Delay\n",
|
||||
"A certificate from the Medical Practitioner recommending the presence in the form of special assistance to be rendered by an additional member during the entire period of hospitalization. The certificate shall also specify the minimum period in which person is admitted in the hospital.\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|Post that in multiple of 50 up to 500|hours|\n",
|
||||
"|500, 750, 1000. Post that in|NIL/100/25|\n",
|
||||
"Discharge summary of the Hospital furnishing details including the date of admission and date of discharge.\n",
|
||||
"\n",
|
||||
"## Trip Cancellation\n",
|
||||
"Stamped boarding pass with invoice used for the travel by the Immediate Family Member.\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|multiple of 1000 till 10000|0/500/100|\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|multiple of 500 till 5000|0/500/100|\n",
|
||||
"\n",
|
||||
"## Trip Interruption\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|multiple of 500 till 10000|0/500/100|\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|multiple of 500 till 5000|0/500/100|\n",
|
||||
"\n",
|
||||
"## Loss of Passport\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|200 to 500 in multiple of 100|Nil|\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100|Independent SI|\n",
|
||||
"\n",
|
||||
"## Loss of International driving License\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|200 to 500 in multiple of 100|Nil|\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100|Independent SI|\n",
|
||||
"\n",
|
||||
"## Missed Connection\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|500 to 1000 in multiple of 100|Nil|\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100|Independent SI|\n",
|
||||
"\n",
|
||||
"## Financial Emergency Cash\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|500 to 1000 in multiple of 100|NIL|\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|10K, 50K, 1Lac, 2Lac, 3Lac, 4Lac|Nil/100/20|\n",
|
||||
"\n",
|
||||
"## Personal Liability\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100 % of Base SI|Part of Base SI|\n",
|
||||
"\n",
|
||||
"## Hijack Daily Allowance\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|25, 50, 75, 100 for 7 days|6/12 hours|\n",
|
||||
"\n",
|
||||
"## Bounced Booking-Hotel/Common Carrier\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100 to 1000 in multiples of 100|6/12 hours|\n",
|
||||
"\n",
|
||||
"|Benefit|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100|Independent SI|\n",
|
||||
"\n",
|
||||
"## Compassionate Visit\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|500 to 5000 in multiple of 500|Nil|\n",
|
||||
"Copy passport of Immediate Family Member with entry and exit stamp.\n",
|
||||
"\n",
|
||||
"## Escort of Minor Child\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|1000 till 5000 in multiple of 1000|Independent SI|\n",
|
||||
"A certificate from the Medical Practitioner specifying the minimum period of Hospitalization.\n",
|
||||
"\n",
|
||||
"## Adventure Sport\n",
|
||||
"Discharge summary of the Hospital furnishing details including the date of admission and date of discharge.\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|100 % of Base SI|Part of Base SI|\n",
|
||||
"Stamped Boarding pass used for the return travel of the child to the Country of Residence.\n",
|
||||
"\n",
|
||||
"## Sports Equipment Hire\n",
|
||||
"Stamped Boarding pass of the attendant from the Country of Residence to the place of hospitalization (if attendant is necessary).\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|500, 1000, 1500 2000|Nil/100|\n",
|
||||
"Copy of passport of the child with entry and exit stamp.\n",
|
||||
"\n",
|
||||
"## Rented sports equipment damage or loss\n",
|
||||
"## Upgradation to Business Class\n",
|
||||
"\n",
|
||||
"|Indemnity|USD|\n",
|
||||
"|---|---|\n",
|
||||
"|500, 1000, 1500 2000|Nil/100|\n",
|
||||
"A certificate from the Medical Practitioner specifying the minimum period of Hospitalization.\n",
|
||||
"\n",
|
||||
"Discharge summary of the Hospital furnishing the details including the date of admission and date of discharge.\n",
|
||||
"\n",
|
||||
"Product Name: Travel infinity | Product UIN: NBHTGBP22011V012223\n",
|
||||
"\n",
|
||||
@@ -497,62 +363,36 @@
|
||||
"\n",
|
||||
"# Insurance Policy\n",
|
||||
"\n",
|
||||
"# Insurance Policy\n",
|
||||
"## Benefits:\n",
|
||||
"\n",
|
||||
"Benefits for Repatriation of Mortal remains is Indemnity. Part of Base SI\n",
|
||||
"\n",
|
||||
"Benefits for Repatriation of Mortal remains is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Total Loss of Checked-in Baggage is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Delay of Checked-in Baggage is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Trip Delay is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Trip Cancellation is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Trip Cancellation is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Trip Interruption is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Trip Interruption is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Loss of Passport is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Loss of Passport is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Loss of International driving License is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Loss of International driving License is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Missed Connection is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Missed Connection is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Financial Emergency Cash is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Personal Liability is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Hijack Daily Allowance is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Bounced Booking-Hotel/Common Carrier is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Bounced Booking-Hotel/Common Carrier is Benefit. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Compassionate Visit is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Escort of Minor Child is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Adventure Sport is Indemnity. Part of Base SI\n",
|
||||
"\n",
|
||||
"Benefits for Sports Equipment Hire is Indemnity. Independent SI\n",
|
||||
"\n",
|
||||
"Benefits for Rented sports equipment damage or loss is Indemnity. Independent SI\n"
|
||||
"- For Inpatient treatment and in this condition when admitted to a hospital, the coverage is reimbursement for medical expenses incurred.\n",
|
||||
"- For Hospital Daily Cash and in this condition when hospitalized, the coverage is daily cash benefit.\n",
|
||||
"- For Waiver of Deductible and in this condition when a deductible is applicable, the coverage is waiver of the deductible amount.\n",
|
||||
"- For Optional Co-payment and in this condition when a co-payment is required, the coverage is optional co-payment.\n",
|
||||
"- For Adventure Sports Cover and in this condition when participating in adventure sports, the coverage is coverage for injuries related to adventure sports.\n",
|
||||
"- For Home to Home Cover and in this condition when requiring medical evacuation, the coverage is assistance for repatriation to home country.\n",
|
||||
"- For Extension to in-patient care and in this condition when extended hospital stay is necessary, the coverage is extension of coverage for in-patient care.\n",
|
||||
"- For Ambulance Charge and in this condition when ambulance services are utilized, the coverage is reimbursement for ambulance charges.\n",
|
||||
"- For Out-patient treatment and in this condition when receiving outpatient medical care, the coverage is reimbursement for outpatient medical expenses.\n",
|
||||
"- For Cancer Screening & Mammographic Examination and in this condition when undergoing cancer screening or mammographic examination, the coverage is coverage for these preventive services.\n",
|
||||
"- For New Born baby Cover and in this condition when a newborn is covered under the policy, the coverage is medical expenses coverage for the newborn.\n",
|
||||
"- For Maternity and in this condition when maternity services are required, the coverage is coverage for maternity expenses.\n",
|
||||
"- For Complete pre-existing disease cover and in this condition when seeking treatment for pre-existing conditions, the coverage is coverage for pre-existing conditions.\n",
|
||||
"- For Medical sum insured replenishment in case of hospitalization due to accident and in this condition when hospitalized due to an accident, the coverage is replenishment of the sum insured.\n",
|
||||
"- For Waiver of sublimit for insured above 60 years of age and in this condition when the insured is above 60 years of age, the coverage is waiver of sublimits.\n",
|
||||
"- For Psychiatric Counseling and in this condition when seeking psychiatric counseling, the coverage is coverage for psychiatric counseling services.\n",
|
||||
"- For Physiotherapy and in this condition when undergoing physiotherapy, the coverage is coverage for physiotherapy sessions.\n",
|
||||
"- For Terrorism cover and in this condition when affected by terrorism, the coverage is coverage for medical expenses related to terrorism incidents.\n",
|
||||
"- For Medical tele-consultation and in this condition when consulting a medical practitioner remotely, the coverage is coverage for tele-consultation services.\n",
|
||||
"- For Medical evacuation and in this condition when requiring medical evacuation, the coverage is coverage for medical evacuation services.\n",
|
||||
"- For Compassionate visit and in this condition when requiring a compassionate visit, the coverage is coverage for travel expenses for a family member to visit.\n",
|
||||
"- For Escort of Minor Child and in this condition when escorting a minor child for medical treatment, the coverage is coverage for escort services for the child.\n",
|
||||
"- For Upgradation to Business Class and in this condition when requiring upgradation to business class for medical travel, the coverage is coverage for upgradation to business class.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"target_page = 51\n",
|
||||
"target_page = 45\n",
|
||||
"pages_vanilla = documents[0].text.split(\"\\n---\\n\")\n",
|
||||
"pages_with_instructions = documents_with_instruction[0].text.split(\"\\n---\\n\")\n",
|
||||
"\n",
|
||||
@@ -563,27 +403,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Embeddings have been explicitly disabled. Using MockEmbedding.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"1it [00:00, 25575.02it/s]\n",
|
||||
"100%|██████████| 1/1 [00:02<00:00, 2.13s/it]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"node_parser_instruction = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)\n",
|
||||
"nodes_instruction = node_parser.get_nodes_from_documents(documents_with_instruction)\n",
|
||||
@@ -604,7 +428,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 22,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -614,9 +438,9 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Vanilla:\n",
|
||||
"You are covered for the expenses you incurred due to the delay of your trip. The amount you paid, which was 45, will be covered up to the limit specified in the certificate of insurance for every block of hours of delay, as mentioned in the policy.\n",
|
||||
"You are covered for the amount you paid due to the trip delay, up to the limit specified in the certificate of insurance.\n",
|
||||
"With instructions:\n",
|
||||
"For Trip Delay coverage, the payment amount for every block of hours of delay is as mentioned in the certificate of insurance.\n"
|
||||
"For Trip Delay coverage, you are covered for a fixed benefit amount as mentioned in the certificate of insurance for every block of hours of delay.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -709,9 +533,9 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"display_name": "llama_parse",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
"name": "llama_parse"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
@@ -723,7 +547,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.6"
|
||||
"version": "3.10.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "W6SX9VAnximx"
|
||||
},
|
||||
"source": [
|
||||
"# LlamaParse With MongoDB\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_mongodb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"In this notebook, we provide a straightforward example of using LlamaParse with MongoDB Atlas VectorSearch.\n",
|
||||
"\n",
|
||||
"We illustrate the process of using llama-parse to parse a PDF document, then index the document with a MongoDB vector store, and subsequently perform basic queries against this store.\n",
|
||||
"\n",
|
||||
"This notebook is structured similarly to quick start guides, aiming to introduce users to utilizing llama-parse in conjunction with a MongoDB Atlas VectorSearch."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "rUJKhWDHxr_k"
|
||||
},
|
||||
"source": [
|
||||
"### Installation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "U6ZkIeBnxfRb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install llama-index llama-parse pip install llama-index-vector-stores-mongodb llama-index-llms-openai"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "wh1eeFJe1gkY"
|
||||
},
|
||||
"source": [
|
||||
"### Setup API Keys"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"id": "I5slpdnyxwIB"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"os.environ[\"LLAMA_CLOUD_API_KEY\"] = '' # Get it from https://cloud.llamaindex.ai/api-key\n",
|
||||
"os.environ['OPENAI_API_KEY'] = '' # Get it from https://platform.openai.com/api-keys"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"id": "es2mz_OVyQw9"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# llama-parse is async-first, running the sync code in a notebook requires the use of nest_asyncio\n",
|
||||
"import nest_asyncio\n",
|
||||
"\n",
|
||||
"nest_asyncio.apply()\n",
|
||||
"\n",
|
||||
"import requests\n",
|
||||
"import pymongo\n",
|
||||
"\n",
|
||||
"from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch\n",
|
||||
"from llama_parse import LlamaParse\n",
|
||||
"from llama_index.embeddings.openai import OpenAIEmbedding\n",
|
||||
"from llama_index.core import VectorStoreIndex, StorageContext\n",
|
||||
"from llama_index.core.node_parser import SimpleNodeParser"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "Ou3bVdHQ10X5"
|
||||
},
|
||||
"source": [
|
||||
"### Download Document\n",
|
||||
"\n",
|
||||
"We will use `Attention is all you need` paper."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "YO9lAk6bybV3",
|
||||
"outputId": "5cee588a-bec5-482e-e8ef-fbb78e8a5967"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Download complete.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# The URL of the file you want to download\n",
|
||||
"url = \"https://arxiv.org/pdf/1706.03762.pdf\"\n",
|
||||
"# The local path where you want to save the file\n",
|
||||
"file_path = \"./attention.pdf\"\n",
|
||||
"\n",
|
||||
"# Perform the HTTP request\n",
|
||||
"response = requests.get(url)\n",
|
||||
"\n",
|
||||
"# Check if the request was successful\n",
|
||||
"if response.status_code == 200:\n",
|
||||
" # Open the file in binary write mode and save the content\n",
|
||||
" with open(file_path, \"wb\") as file:\n",
|
||||
" file.write(response.content)\n",
|
||||
" print(\"Download complete.\")\n",
|
||||
"else:\n",
|
||||
" print(\"Error downloading the file.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "1NtR7PGo13Hh"
|
||||
},
|
||||
"source": [
|
||||
"### Parse the document using `LlamaParse`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "reeJsblfyeSd",
|
||||
"outputId": "bb569e9f-fe31-47b9-a059-d7da369b3f94"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 09a49745-9f21-4190-9de8-27e4e1a4bdf5\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"documents = LlamaParse(result_type=\"text\").load_data(file_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "-NIXtCBwyiPp",
|
||||
"outputId": "ad4b3cec-2c23-4858-81f0-994ae2c96b8f"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"rmer - model architecture.\n",
|
||||
"The Transformer follows this overall architecture using stacked self-attention and point-wise, fully\n",
|
||||
"connected layers for both the encoder and decoder, shown in the left and right halves of Figure 1,\n",
|
||||
"respectively.\n",
|
||||
"3.1 Encoder and Decoder Stacks\n",
|
||||
"Encoder: The encoder is composed of a stack of N = 6 identical layers. Each layer has two\n",
|
||||
"sub-layers. The first is a multi-head self-attention mechanism, and the second is a simple, position-\n",
|
||||
"wise fully connected feed-forward network. We employ a residual connection [11] around each of\n",
|
||||
"the two sub-layers, followed by layer normalization [1]. That is, the output of each sub-layer is\n",
|
||||
"LayerNorm(x + Sublayer(x)), where Sublayer(x) is the function implemented by the sub-layer\n",
|
||||
"itself. To facilitate these residual connections, all sub-layers in the model, as well as the embedding\n",
|
||||
"layers, produce outputs of dimension dmodel = 512.\n",
|
||||
"Decoder: The decoder is also composed of a stack of N = 6 identical layers. In addition \n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Take a quick look at some of the parsed text from the document:\n",
|
||||
"print(documents[0].get_content()[10000:11000])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "wP9I5dhB1-w1"
|
||||
},
|
||||
"source": [
|
||||
"### Create `MongoDBAtlasVectorSearch`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"id": "-4Ek0oK-yp3L"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"mongo_uri = os.environ[\"MONGO_URI\"]\n",
|
||||
"\n",
|
||||
"mongodb_client = pymongo.MongoClient(mongo_uri)\n",
|
||||
"mongodb_vector_store = MongoDBAtlasVectorSearch(mongodb_client)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "GYiVwFok2DNf"
|
||||
},
|
||||
"source": [
|
||||
"### Create nodes."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"id": "aqdF6ZonytHF"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"node_parser = SimpleNodeParser()\n",
|
||||
"\n",
|
||||
"nodes = node_parser.get_nodes_from_documents(documents)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "U5fMoGrA2GSH"
|
||||
},
|
||||
"source": [
|
||||
"### Create Index and Query Engine."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"id": "gQUieIrAywSC"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"storage_context = StorageContext.from_defaults(vector_store=mongodb_vector_store)\n",
|
||||
"\n",
|
||||
"index = VectorStoreIndex(\n",
|
||||
" nodes=nodes,\n",
|
||||
" storage_context=storage_context,\n",
|
||||
" embed_model=OpenAIEmbedding(),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {
|
||||
"id": "snkZZss-zKDb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query_engine = index.as_query_engine(similarity_top_k=2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "rTKT34XO2LYk"
|
||||
},
|
||||
"source": [
|
||||
"### Test Query"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "r66ciuPkzNv1",
|
||||
"outputId": "919218e3-0884-4992-802c-ab1c4622ec4b"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"***********New LlamaParse+ Basic Query Engine***********\n",
|
||||
"The BLEU score on the WMT 2014 English-to-German translation task is 28.4.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"query = \"What is BLEU score on the WMT 2014 English-to-German translation task?\"\n",
|
||||
"\n",
|
||||
"response = query_engine.query(query)\n",
|
||||
"print(\"\\n***********New LlamaParse+ Basic Query Engine***********\")\n",
|
||||
"print(response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "K7RsivpwzQBo",
|
||||
"outputId": "9bcbf62e-250c-46db-f247-e1f293c09bbe"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"We varied the learning\n",
|
||||
"rate over the course of training, according to the formula:\n",
|
||||
" lrate = d−0.5 (3)\n",
|
||||
" model · min(step_num−0.5, step_num · warmup_steps−1.5)\n",
|
||||
"This corresponds to increasing the learning rate linearly for the first warmup_steps training steps,\n",
|
||||
"and decreasing it thereafter proportionally to the inverse square root of the step number. We used\n",
|
||||
"warmup_steps = 4000.\n",
|
||||
"5.4 Regularization\n",
|
||||
"We employ three types of regularization during training:\n",
|
||||
" 7\n",
|
||||
"---\n",
|
||||
"Table 2: The Transformer achieves better BLEU scores than previous state-of-the-art models on the\n",
|
||||
"English-to-German and English-to-French newstest2014 tests at a fraction of the training cost.\n",
|
||||
" Model BLEU Training Cost (FLOPs)\n",
|
||||
" EN-DE EN-FR EN-DE EN-FR\n",
|
||||
" ByteNet [18] 23.75\n",
|
||||
" Deep-Att + PosUnk [39] 39.2 1.0 · 1020\n",
|
||||
" GNMT + RL [38] 24.6 39.92 2.3 · 1019 1.4 · 1020\n",
|
||||
" ConvS2S [9] 25.16 40.46 9.6 · 1018 1.5 · 1020\n",
|
||||
" MoE [32] 26.03 40.56 2.0 · 1019 1.2 · 1020\n",
|
||||
" Deep-Att + PosUnk Ensemble [39] 40.4 8.0 · 1020\n",
|
||||
" GNMT + RL Ensemble [38] 26.30 41.16 1.8 · 1020 1.1 · 1021\n",
|
||||
" ConvS2S Ensemble [9] 26.36 41.29 7.7 · 1019 1.2 · 1021\n",
|
||||
" Transformer (base model) 27.3 38.1 3.3 · 1018\n",
|
||||
" Transformer (big) 28.4 41.8 2.3 · 1019\n",
|
||||
"Residual Dropout We apply dropout [33] to the output of each sub-layer, before it is added to the\n",
|
||||
"sub-layer input and normalized. In addition, we apply dropout to the sums of the embeddings and the\n",
|
||||
"positional encodings in both the encoder and decoder stacks. For the base model, we use a rate of\n",
|
||||
"Pdrop = 0.1.\n",
|
||||
"Label Smoothing During training, we employed label smoothing of value ϵls = 0.1 [36]. This\n",
|
||||
"hurts perplexity, as the model learns to be more unsure, but improves accuracy and BLEU score.\n",
|
||||
"6 Results\n",
|
||||
"6.1 Machine Translation\n",
|
||||
"On the WMT 2014 English-to-German translation task, the big transformer model (Transformer (big)\n",
|
||||
"in Table 2) outperforms the best previously reported models (including ensembles) by more than 2.0\n",
|
||||
"BLEU, establishing a new state-of-the-art BLEU score of 28.4. The configuration of this model is\n",
|
||||
"listed in the bottom line of Table 3. Training took 3.5 days on 8 P100 GPUs. Even our base model\n",
|
||||
"surpasses all previously published models and ensembles, at a fraction of the training cost of any of\n",
|
||||
"the competitive models.\n",
|
||||
"On the WMT 2014 English-to-French translation task, our big model achieves a BLEU score of 41.0,\n",
|
||||
"outperforming all of the previously published single models, at less than 1/4 the training cost of the\n",
|
||||
"previous state-of-the-art model. The Transformer (big) model trained for English-to-French used\n",
|
||||
"dropout rate Pdrop = 0.1, instead of 0.3.\n",
|
||||
"For the base models, we used a single model obtained by averaging the last 5 checkpoints, which\n",
|
||||
"were written at 10-minute intervals. For the big models, we averaged the last 20 checkpoints. We\n",
|
||||
"used beam search with a beam size of 4 and length penalty α = 0.6 [38]. These hyperparameters\n",
|
||||
"were chosen after experimentation on the development set. We set the maximum output length during\n",
|
||||
"inference to input length + 50, but terminate early when possible [38].\n",
|
||||
"Table 2 summarizes our results and compares our translation quality and training costs to other model\n",
|
||||
"architectures from the literature.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Take a look at one of the source nodes from the response\n",
|
||||
"print(response.source_nodes[0].get_content())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "anthropic_env",
|
||||
"language": "python",
|
||||
"name": "anthropic_env"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.3"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
+42
-6
@@ -110,17 +110,53 @@ class Language(str, Enum):
|
||||
|
||||
SUPPORTED_FILE_TYPES = [
|
||||
".pdf",
|
||||
".xml"
|
||||
".602",
|
||||
".abw",
|
||||
".cgm",
|
||||
".cwk",
|
||||
".doc",
|
||||
".docx",
|
||||
".pptx",
|
||||
".rtf",
|
||||
".pages",
|
||||
".docm",
|
||||
".dot",
|
||||
".dotm",
|
||||
".hwp",
|
||||
".key",
|
||||
".epub"
|
||||
".lwp",
|
||||
".mw",
|
||||
".mcw",
|
||||
".pages",
|
||||
".pbd",
|
||||
".ppt",
|
||||
".pptm",
|
||||
".pptx",
|
||||
".pot",
|
||||
".potm",
|
||||
".potx",
|
||||
".rtf",
|
||||
".sda",
|
||||
".sdd",
|
||||
".sdp",
|
||||
".sdw",
|
||||
".sgl",
|
||||
".sti",
|
||||
".sxi",
|
||||
".sxw",
|
||||
".stw",
|
||||
".sxg",
|
||||
".txt",
|
||||
".uof",
|
||||
".uop",
|
||||
".uot",
|
||||
".vor",
|
||||
".wpd",
|
||||
".wps",
|
||||
".xml",
|
||||
".zabw",
|
||||
".epub",
|
||||
".htm",
|
||||
".html"
|
||||
]
|
||||
|
||||
|
||||
class LlamaParse(BasePydanticReader):
|
||||
"""A smart-parser for files."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user