neuml/txtai
1
0
Fork 0
mirror of https://github.com/neuml/txtai.git synced 2026-10-11 21:39:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update notebook

Browse Source
This commit is contained in:
davidmezzetti
2023-05-16 09:49:23 -04:00
parent 1c95b2d546
commit 4f359a2c8a
1 changed files with 130 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/45_Customize_your_own_embeddings_database.ipynb
+130 -78
View File
@@ -44,7 +44,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 39,
"metadata": {
"id": "xEvX9vCpn4E0"
},
@@ -109,7 +109,7 @@
"metadata": {
"id": "DMqiTrTbC-VJ"
},
"execution_count": null,
"execution_count": 41,
"outputs": []
},
{
@@ -122,9 +122,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "hcAcJikVDMNQ",
"outputId": "87b3b4d6-ab76-435e-c2c8-1107cd574d8d"
"outputId": "a587620a-5657-4082-84ee-3d73114e1d3a"
},
"execution_count": null,
"execution_count": 42,
"outputs": [
{
"output_type": "execute_result",
@@ -136,7 +136,7 @@
]
},
"metadata": {},
"execution_count": 76
"execution_count": 42
}
]
},
@@ -150,9 +150,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "FQYx76IgMinE",
"outputId": "b97538e6-ffb7-44dc-92f2-8a00f60557d1"
"outputId": "9375bf2b-e641-4b01-cc8a-b400c5baf399"
},
"execution_count": null,
"execution_count": 43,
"outputs": [
{
"output_type": "stream",
@@ -161,7 +161,7 @@
"{\n",
" \"backend\": \"numpy\",\n",
" \"build\": {\n",
" \"create\": \"2023-05-04T12:12:02Z\",\n",
" \"create\": \"2023-05-16T13:38:32Z\",\n",
" \"python\": \"3.10.11\",\n",
" \"settings\": {\n",
" \"numpy\": \"1.22.4\"\n",
@@ -172,7 +172,7 @@
" \"dimensions\": 384,\n",
" \"offset\": 25000,\n",
" \"path\": \"sentence-transformers/all-MiniLM-L6-v2\",\n",
" \"update\": \"2023-05-04T12:12:02Z\"\n",
" \"update\": \"2023-05-16T13:38:32Z\"\n",
"}\n"
]
}
@@ -210,7 +210,7 @@
"metadata": {
"id": "DPWrubv5oOn7"
},
"execution_count": null,
"execution_count": 44,
"outputs": []
},
{
@@ -232,9 +232,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "ILSfWHxVHex0",
"outputId": "b302ef06-05ad-4082-da5e-bcde73273108"
"outputId": "f4cc3f44-da63-4be9-9187-58386fad58df"
},
"execution_count": null,
"execution_count": 45,
"outputs": [
{
"output_type": "execute_result",
@@ -252,7 +252,7 @@
]
},
"metadata": {},
"execution_count": 79
"execution_count": 45
}
]
},
@@ -266,9 +266,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "0IuVqFxUMwe8",
"outputId": "f91dfe5a-814c-4492-8cb9-88ece318a409"
"outputId": "fa13d767-d549-4b9c-f63e-7e09b05159ee"
},
"execution_count": null,
"execution_count": 46,
"outputs": [
{
"output_type": "stream",
@@ -277,7 +277,7 @@
"{\n",
" \"backend\": \"numpy\",\n",
" \"build\": {\n",
" \"create\": \"2023-05-04T12:12:24Z\",\n",
" \"create\": \"2023-05-16T13:38:52Z\",\n",
" \"python\": \"3.10.11\",\n",
" \"settings\": {\n",
" \"numpy\": \"1.22.4\"\n",
@@ -289,7 +289,7 @@
" \"dimensions\": 384,\n",
" \"offset\": 25000,\n",
" \"path\": \"sentence-transformers/all-MiniLM-L6-v2\",\n",
" \"update\": \"2023-05-04T12:12:24Z\"\n",
" \"update\": \"2023-05-16T13:38:52Z\"\n",
"}\n"
]
}
@@ -317,9 +317,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "c2FVQlxSLKgP",
"outputId": "43bce3a2-6aae-44fb-9db1-2c650df0507c"
"outputId": "1b8e454d-c5f7-4d58-b2b0-a1a9eaca4b9b"
},
"execution_count": null,
"execution_count": 47,
"outputs": [
{
"output_type": "stream",
@@ -350,15 +350,15 @@
"base_uri": "https://localhost:8080/"
},
"id": "JvodDi4w6JxS",
"outputId": "0427d12e-9569-4c7a-d24e-4860cbbe6567"
"outputId": "3671753d-0dd1-47fe-bef6-04841204cb6b"
},
"execution_count": null,
"execution_count": 48,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0.03392000120000011 seconds per query\n"
"0.028768999292000445 seconds per query\n"
]
}
]
@@ -396,7 +396,7 @@
"metadata": {
"id": "OYAqPoTmNaNN"
},
"execution_count": null,
"execution_count": 49,
"outputs": []
},
{
@@ -418,9 +418,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "zvlrEunM7vi4",
"outputId": "20b0ed74-9355-4bd1-befb-6646a3dd6c83"
"outputId": "e591495f-0622-4291-fbbd-cbe655f92a07"
},
"execution_count": null,
"execution_count": 50,
"outputs": [
{
"output_type": "execute_result",
@@ -438,7 +438,7 @@
]
},
"metadata": {},
"execution_count": 84
"execution_count": 50
}
]
},
@@ -452,9 +452,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "QmJZb56SM6Up",
"outputId": "d9dd3a3d-e574-4f59-b575-bf0fca2db31f"
"outputId": "a6e9b73a-6248-43ca-a57f-c3a4fec9112f"
},
"execution_count": null,
"execution_count": 51,
"outputs": [
{
"output_type": "stream",
@@ -463,7 +463,7 @@
"{\n",
" \"backend\": \"torch\",\n",
" \"build\": {\n",
" \"create\": \"2023-05-04T12:12:53Z\",\n",
" \"create\": \"2023-05-16T13:39:19Z\",\n",
" \"python\": \"3.10.11\",\n",
" \"settings\": {\n",
" \"torch\": \"2.0.0+cu118\"\n",
@@ -475,7 +475,7 @@
" \"dimensions\": 384,\n",
" \"offset\": 25000,\n",
" \"path\": \"sentence-transformers/all-MiniLM-L6-v2\",\n",
" \"update\": \"2023-05-04T12:12:53Z\"\n",
" \"update\": \"2023-05-16T13:39:19Z\"\n",
"}\n"
]
}
@@ -501,9 +501,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "u5JFEJ-q5Zow",
"outputId": "c69f084b-7958-404f-e987-afc7ac34b7d8"
"outputId": "fffb3e3d-2d5c-4f20-877a-cce52836e5f3"
},
"execution_count": null,
"execution_count": 52,
"outputs": [
{
"output_type": "stream",
@@ -534,15 +534,15 @@
"base_uri": "https://localhost:8080/"
},
"id": "7aAI_jUm6goL",
"outputId": "c0661412-d99a-48fa-c926-f120394a8b60"
"outputId": "e4ec71dc-c1d8-40ff-9e68-113a5da1ad1e"
},
"execution_count": null,
"execution_count": 53,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0.021084972200000267 seconds per query\n"
"0.0198183048359997 seconds per query\n"
]
}
]
@@ -579,7 +579,7 @@
"metadata": {
"id": "DQECU7y-9doj"
},
"execution_count": null,
"execution_count": 54,
"outputs": []
},
{
@@ -592,9 +592,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "wGqcmE6M9kJW",
"outputId": "e2ef30e8-cc9e-4909-ae53-2bb171b3cd9e"
"outputId": "25c7ef4c-a437-4d64-ebaf-1481d0873ca1"
},
"execution_count": null,
"execution_count": 55,
"outputs": [
{
"output_type": "execute_result",
@@ -612,7 +612,7 @@
]
},
"metadata": {},
"execution_count": 89
"execution_count": 55
}
]
},
@@ -626,9 +626,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "zDxqlMT9d-Q3",
"outputId": "5181aed0-f16a-4af0-f51a-06678be4aaaf"
"outputId": "befdb0a1-01b4-4948-94e2-4570efada3ce"
},
"execution_count": null,
"execution_count": 56,
"outputs": [
{
"output_type": "stream",
@@ -637,7 +637,7 @@
"{\n",
" \"backend\": \"faiss\",\n",
" \"build\": {\n",
" \"create\": \"2023-05-04T12:13:23Z\",\n",
" \"create\": \"2023-05-16T13:39:47Z\",\n",
" \"python\": \"3.10.11\",\n",
" \"settings\": {\n",
" \"components\": \"IVF632,Flat\"\n",
@@ -649,7 +649,7 @@
" \"dimensions\": 384,\n",
" \"offset\": 25000,\n",
" \"path\": \"sentence-transformers/all-MiniLM-L6-v2\",\n",
" \"update\": \"2023-05-04T12:13:23Z\"\n",
" \"update\": \"2023-05-16T13:39:47Z\"\n",
"}\n"
]
}
@@ -665,15 +665,15 @@
"base_uri": "https://localhost:8080/"
},
"id": "zlbc43qg9qKb",
"outputId": "327984fe-fc27-4eb9-faf1-feb84c73bd68"
"outputId": "ae06092c-ff5f-4540-afd5-e054db6ffb84"
},
"execution_count": null,
"execution_count": 57,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0.008729957724000087 seconds per query\n"
"0.00825659705599992 seconds per query\n"
]
}
]
@@ -710,7 +710,7 @@
"metadata": {
"id": "5dqxj2hr_ICl"
},
"execution_count": null,
"execution_count": 58,
"outputs": []
},
{
@@ -723,9 +723,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "pUw-3WCHFGf9",
"outputId": "67ff8950-3570-4f53-f712-72f0feb27b81"
"outputId": "22e827a9-ed34-4847-b41b-70499f85cff0"
},
"execution_count": null,
"execution_count": 59,
"outputs": [
{
"output_type": "execute_result",
@@ -733,17 +733,17 @@
"text/plain": [
"[{'id': '19831',\n",
" 'text': 'Boston Red Sox Team Report - September 6 (Sports Network) - Two of the top teams in the American League tangle in a possible American League Division Series preview tonight, as the West-leading Oakland Athletics host the wild card-leading Boston Red Sox for the first of a three-game set at the ',\n",
" 'score': 0.6780003309249878},\n",
" 'score': 0.678000271320343},\n",
" {'id': '18302',\n",
" 'text': 'BASEBALL: RED-HOT SOX CLIP THE ANGELS #39; WINGS BOSTON RED SOX fans are enjoying their best week of the season. While their beloved team swept wild-card rivals Anaheim in a three-game series to establish a nine-game winning streak, the hated New York Yankees endured the heaviest loss in their history.',\n",
" 'score': 0.6639198660850525},\n",
" 'score': 0.6639199256896973},\n",
" {'id': '16370',\n",
" 'text': 'Boston Red Sox Team Report - September 1 (Sports Network) - The red-hot Boston Red Sox hope to continue rolling as they continue their three-game set with the Anaheim Angels this evening at Fenway Park.',\n",
" 'score': 0.6617192029953003}]"
" 'score': 0.6617191433906555}]"
]
},
"metadata": {},
"execution_count": 93
"execution_count": 59
}
]
},
@@ -757,9 +757,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "u5DAfB1MeCLF",
"outputId": "b7a78b99-1bc8-4ae3-9a3d-ab2b8346dfbb"
"outputId": "d00dec7f-e1da-49f1-afd3-aa852238769f"
},
"execution_count": null,
"execution_count": 60,
"outputs": [
{
"output_type": "stream",
@@ -768,7 +768,7 @@
"{\n",
" \"backend\": \"hnsw\",\n",
" \"build\": {\n",
" \"create\": \"2023-05-04T12:13:59Z\",\n",
" \"create\": \"2023-05-16T13:40:21Z\",\n",
" \"python\": \"3.10.11\",\n",
" \"settings\": {\n",
" \"efconstruction\": 200,\n",
@@ -787,7 +787,7 @@
" \"metric\": \"ip\",\n",
" \"offset\": 25000,\n",
" \"path\": \"sentence-transformers/all-MiniLM-L6-v2\",\n",
" \"update\": \"2023-05-04T12:13:59Z\"\n",
" \"update\": \"2023-05-16T13:40:21Z\"\n",
"}\n"
]
}
@@ -803,15 +803,15 @@
"base_uri": "https://localhost:8080/"
},
"id": "Z3Qg6EEjFIom",
"outputId": "7b46e190-f60d-495f-d813-7a3d9a5c471d"
"outputId": "49d01a91-5d53-4792-8244-316b6e79cc20"
},
"execution_count": null,
"execution_count": 61,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0.006160191656000279 seconds per query\n"
"0.006280380824000531 seconds per query\n"
]
}
]
@@ -827,6 +827,58 @@
"id": "JREMWY5NHAX-"
}
},
{
"cell_type": "markdown",
"source": [
"# External Vectorization\n",
"\n",
"txtai has a number of built-in vectorizers backed by Hugging Face Transformers and Sentence Transformers. Just like other txtai modules, vectorization can also be customized.\n",
"\n",
"The next example uses the Hugging Face Inference API to vectorize text.\n"
],
"metadata": {
"id": "Wuj_gZeBs57O"
}
},
{
"cell_type": "code",
"source": [
"import numpy as np\n",
"import requests\n",
"\n",
"BASE = \"https://api-inference.huggingface.co/pipeline/feature-extraction\"\n",
"\n",
"def transform(inputs):\n",
" # Your API provider of choice\n",
" response = requests.post(f\"{BASE}/sentence-transformers/all-MiniLM-L6-v2\", json={\"inputs\": inputs})\n",
" return np.array(response.json(), dtype=np.float32)\n",
"\n",
"embeddings = Embeddings({\"transform\": transform, \"backend\": \"numpy\", \"content\": True})\n",
"embeddings.index([(0, \"sunny\", None), (1, \"rainy\", None)])\n",
"embeddings.search(\"nice day\") "
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "FAhg1TcdtWIJ",
"outputId": "22e34bd0-b6d3-40b2-8948-d08397d744f5"
},
"execution_count": 62,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[{'id': '0', 'text': 'sunny', 'score': 0.28077083826065063},\n",
" {'id': '1', 'text': 'rainy', 'score': 0.18051263689994812}]"
]
},
"metadata": {},
"execution_count": 62
}
]
},
{
"cell_type": "markdown",
"source": [
@@ -857,9 +909,9 @@
"base_uri": "https://localhost:8080/"
},
"id": "X1DYuyPmmSgU",
"outputId": "dc4d406a-b60b-43f7-cc38-d7787582567d"
"outputId": "975c0f7f-f38e-478e-a8b8-16155f1f4865"
},
"execution_count": null,
"execution_count": 63,
"outputs": [
{
"output_type": "stream",
@@ -873,7 +925,7 @@
" \"backend\": \"faiss\",\n",
" \"offset\": 25000,\n",
" \"build\": {\n",
" \"create\": \"2023-05-04T12:14:25Z\",\n",
" \"create\": \"2023-05-16T13:40:49Z\",\n",
" \"python\": \"3.10.11\",\n",
" \"settings\": {\n",
" \"components\": \"IVF632,Flat\"\n",
@@ -881,7 +933,7 @@
" \"system\": \"Linux (x86_64)\",\n",
" \"txtai\": \"5.6.0\"\n",
" },\n",
" \"update\": \"2023-05-04T12:14:25Z\"\n",
" \"update\": \"2023-05-16T13:40:49Z\"\n",
"}"
]
}
@@ -921,7 +973,7 @@
"metadata": {
"id": "KZ-x_53SHsNK"
},
"execution_count": null,
"execution_count": 64,
"outputs": []
},
{
@@ -934,15 +986,15 @@
"base_uri": "https://localhost:8080/"
},
"id": "LQZCGu-9H70K",
"outputId": "431dd7d4-7393-488e-9e67-b986967223a2"
"outputId": "06e41f55-687d-4d98-e194-13efdd2991ef"
},
"execution_count": null,
"execution_count": 65,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0.0001413383999997677 seconds per query\n"
"0.00012401376399975562 seconds per query\n"
]
}
]
@@ -957,15 +1009,15 @@
"base_uri": "https://localhost:8080/"
},
"id": "SwgR2TwPHvdP",
"outputId": "b057c00c-7b47-443b-9bc3-058d1f64621b"
"outputId": "d88a457a-7789-4881-9420-914e2992d132"
},
"execution_count": null,
"execution_count": 66,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0.03718761139199978 seconds per query\n"
"0.03863514600000053 seconds per query\n"
]
}
]
@@ -982,7 +1034,7 @@
"metadata": {
"id": "ZdrLBOmaIKbF"
},
"execution_count": null,
"execution_count": 67,
"outputs": []
},
{
@@ -995,15 +1047,15 @@
"base_uri": "https://localhost:8080/"
},
"id": "cce1PAVqIMpU",
"outputId": "1d3e0f69-e88c-420b-caff-a20873436cba"
"outputId": "51b8749b-7a87-42d0-af17-2710f8460c68"
},
"execution_count": null,
"execution_count": 68,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0.002780103128000519 seconds per query\n"
"0.0038918176440001844 seconds per query\n"
]
}
]
@@ -1018,15 +1070,15 @@
"base_uri": "https://localhost:8080/"
},
"id": "lxBfoh3TINmE",
"outputId": "eab76ed5-a619-4443-f29e-3d79cfa57b2d"
"outputId": "f56ed46b-abb6-4f6f-e986-10ec570e8cbe"
},
"execution_count": null,
"execution_count": 69,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0.01854579007600023 seconds per query\n"
"0.0198518766039997 seconds per query\n"
]
}
]