feat: add AWS Bedrock as LLM

backend/.env

@@ -1,6 +1,8 @@
+# AWS Bedrock profile name
+# AWS_PROFILE_NAME=
+
 # The name of LLM model to use.
-# MODEL=amazon.titan-text-express-v1
-MODEL=gpt-3.5-turbo
+MODEL=amazon.titan-text-express-v1
 
 # The OpenAI API key to use.
 # OPENAI_API_KEY=

backend/app/settings.py

@@ -1,7 +1,7 @@
 import os
 from typing import Dict
 from llama_index.core.settings import Settings
-from llama_index.llms.openai import OpenAI
+from llama_index.llms.bedrock import Bedrock
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 
 
@@ -11,11 +11,13 @@ def llm_config_from_env() -> Dict:
     model = os.getenv("MODEL")
     temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
     max_tokens = os.getenv("LLM_MAX_TOKENS")
+    profile_name = os.getenv("AWS_PROFILE_NAME")
 
     config = {
         "model": model,
         "temperature": float(temperature),
         "max_tokens": int(max_tokens) if max_tokens is not None else None,
+        "profile_name": profile_name,
     }
     return config
 
@@ -31,7 +33,7 @@ def init_settings():
     llm_configs = llm_config_from_env()
     embedding_configs = embedding_config_from_env()
 
-    Settings.llm = OpenAI(**llm_configs)
+    Settings.llm = Bedrock(**llm_configs)
     Settings.embed_model = HuggingFaceEmbedding(**embedding_configs)
     Settings.chunk_size = int(os.getenv("CHUNK_SIZE", "1024"))
     Settings.chunk_overlap = int(os.getenv("CHUNK_OVERLAP", "20"))

backend/poetry.lock

@@ -120,6 +120,30 @@ files = [
     {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"},
 ]
 
+[[package]]
+name = "anthropic"
+version = "0.17.0"
+description = "The official Python library for the anthropic API"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "anthropic-0.17.0-py3-none-any.whl", hash = "sha256:e582635d65d940cd0d957631d548aa2d8add4f261556bf53f8c1f6e5fa6082fd"},
+    {file = "anthropic-0.17.0.tar.gz", hash = "sha256:6f1958e7ffd706a19741260d6dfef3fd9af07d31a57b837792331b4ba0aa067c"},
+]
+
+[package.dependencies]
+anyio = ">=3.5.0,<5"
+distro = ">=1.7.0,<2"
+httpx = ">=0.23.0,<1"
+pydantic = ">=1.9.0,<3"
+sniffio = "*"
+tokenizers = ">=0.13.0"
+typing-extensions = ">=4.7,<5"
+
+[package.extras]
+bedrock = ["boto3 (>=1.28.57)", "botocore (>=1.31.57)"]
+vertex = ["google-auth (>=2,<3)"]
+
 [[package]]
 name = "anyio"
 version = "4.3.0"
@@ -245,6 +269,44 @@ charset-normalizer = ["charset-normalizer"]
 html5lib = ["html5lib"]
 lxml = ["lxml"]
 
+[[package]]
+name = "boto3"
+version = "1.34.60"
+description = "The AWS SDK for Python"
+optional = false
+python-versions = ">= 3.8"
+files = [
+    {file = "boto3-1.34.60-py3-none-any.whl", hash = "sha256:199f11518b370e2ec1f6d5ec0c647eca967b6eb2cf6a332d9fc01a2144ea3690"},
+    {file = "boto3-1.34.60.tar.gz", hash = "sha256:f78f30f4d6b1d5839ec20da9ffe0f3b36c1b404a415d208459a0b88c202a05e9"},
+]
+
+[package.dependencies]
+botocore = ">=1.34.60,<1.35.0"
+jmespath = ">=0.7.1,<2.0.0"
+s3transfer = ">=0.10.0,<0.11.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
+
+[[package]]
+name = "botocore"
+version = "1.34.60"
+description = "Low-level, data-driven core of boto 3."
+optional = false
+python-versions = ">= 3.8"
+files = [
+    {file = "botocore-1.34.60-py3-none-any.whl", hash = "sha256:6e5317aab5dea19579e7c33528d53761bfb02f3fac5da2de01d1686a25f116a5"},
+    {file = "botocore-1.34.60.tar.gz", hash = "sha256:4101494f0b692c95c592cba2719a61854e1c2923d89c60eaddf0e0d986442562"},
+]
+
+[package.dependencies]
+jmespath = ">=0.7.1,<2.0.0"
+python-dateutil = ">=2.1,<3.0.0"
+urllib3 = {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}
+
+[package.extras]
+crt = ["awscrt (==0.19.19)"]
+
 [[package]]
 name = "bs4"
 version = "0.0.2"
@@ -1145,6 +1207,17 @@ MarkupSafe = ">=2.0"
 [package.extras]
 i18n = ["Babel (>=2.7)"]
 
+[[package]]
+name = "jmespath"
+version = "1.0.1"
+description = "JSON Matching Expressions"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
+    {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
+]
+
 [[package]]
 name = "joblib"
 version = "1.3.2"
@@ -1367,6 +1440,37 @@ local-models = ["optimum[onnxruntime] (>=1.13.2,<2.0.0)", "sentencepiece (>=0.1.
 postgres = ["asyncpg (>=0.28.0,<0.29.0)", "pgvector (>=0.1.0,<0.2.0)", "psycopg2-binary (>=2.9.9,<3.0.0)"]
 query-tools = ["guidance (>=0.0.64,<0.0.65)", "jsonpath-ng (>=1.6.0,<2.0.0)", "lm-format-enforcer (>=0.4.3,<0.5.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "scikit-learn", "spacy (>=3.7.1,<4.0.0)"]
 
+[[package]]
+name = "llama-index-llms-anthropic"
+version = "0.1.5"
+description = "llama-index llms anthropic integration"
+optional = false
+python-versions = ">=3.8.1,<4.0"
+files = [
+    {file = "llama_index_llms_anthropic-0.1.5-py3-none-any.whl", hash = "sha256:4884fa25c07073baec248aa07b843c29c969a8515dec58abda86cd8e5e15f44f"},
+    {file = "llama_index_llms_anthropic-0.1.5.tar.gz", hash = "sha256:33d4622d7833d8ca326cd57e903ad50da3efbf7ae92bdf70b37b4b5419d8c91f"},
+]
+
+[package.dependencies]
+anthropic = ">=0.17.0,<0.18.0"
+llama-index-core = ">=0.10.1,<0.11.0"
+
+[[package]]
+name = "llama-index-llms-bedrock"
+version = "0.1.3"
+description = "llama-index llms bedrock integration"
+optional = false
+python-versions = ">=3.8.1,<4.0"
+files = [
+    {file = "llama_index_llms_bedrock-0.1.3-py3-none-any.whl", hash = "sha256:2b2637490c56cb2fda234e3047168afcf0bde812ee9954fa184c87534b1aec95"},
+    {file = "llama_index_llms_bedrock-0.1.3.tar.gz", hash = "sha256:4c8d2a0687aea3064b3f01e6b3d526d01f91f5e9a3a75c3a15ed577c429cf700"},
+]
+
+[package.dependencies]
+boto3 = ">=1.34.26,<2.0.0"
+llama-index-core = ">=0.10.1,<0.11.0"
+llama-index-llms-anthropic = ">=0.1.1,<0.2.0"
+
 [[package]]
 name = "llama-index-llms-openai"
 version = "0.1.7"
@@ -3131,6 +3235,23 @@ files = [
 [package.dependencies]
 pyasn1 = ">=0.1.3"
 
+[[package]]
+name = "s3transfer"
+version = "0.10.0"
+description = "An Amazon S3 Transfer Manager"
+optional = false
+python-versions = ">= 3.8"
+files = [
+    {file = "s3transfer-0.10.0-py3-none-any.whl", hash = "sha256:3cdb40f5cfa6966e812209d0994f2a4709b561c88e90cf00c2696d2df4e56b2e"},
+    {file = "s3transfer-0.10.0.tar.gz", hash = "sha256:d0c8bbf672d5eebbe4e57945e23b972d963f07d82f661cabf678a5c88831595b"},
+]
+
+[package.dependencies]
+botocore = ">=1.33.2,<2.0a.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"]
+
 [[package]]
 name = "safetensors"
 version = "0.4.2"
@@ -4331,4 +4452,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11,<3.12"
-content-hash = "3f14c198f20429f69d2c61b2b554b94d5257292a672f0f90022cc4fa34e2ba89"
+content-hash = "5f19f7fdb8eaf942bf6e55077603475a2b1d958a24f8d429590cc7b723faec62"

backend/pyproject.toml

@@ -12,6 +12,7 @@ fastapi = "^0.109.1"
 python-dotenv = "^1.0.0"
 llama-index = "^0.10.7"
 llama-index-embeddings-huggingface = "^0.1.4"
+llama-index-llms-bedrock = "^0.1.3"
 
 [tool.poetry.dependencies.uvicorn]
 extras = [ "standard" ]