RELEASING: Releasing 1 package(s)

Releases:
  llamaindex@0.1.19

[skip ci]

apps/docs/docs/modules/evaluation/modules/correctness.md

@@ -53,10 +53,6 @@ const evaluator = new CorrectnessEvaluator({
   serviceContext: ctx,
 });
 
-const response = await queryEngine.query({
-  query,
-});
-
 const result = await evaluator.evaluateResponse({
   query,
   response,

examples/agent/queryOpenAIAgent.ts

@@ -1,46 +0,0 @@
-import {
-  OpenAIAgent,
-  QueryEngineTool,
-  SimpleDirectoryReader,
-  VectorStoreIndex,
-} from "llamaindex";
-
-async function main() {
-  // Load the documents
-  const documents = await new SimpleDirectoryReader().loadData({
-    directoryPath: "node_modules/llamaindex/examples/",
-  });
-
-  // Create a vector index from the documents
-  const vectorIndex = await VectorStoreIndex.fromDocuments(documents);
-
-  // Create a query engine from the vector index
-  const abramovQueryEngine = vectorIndex.asQueryEngine();
-
-  // Create a QueryEngineTool with the query engine
-  const queryEngineTool = new QueryEngineTool({
-    queryEngine: abramovQueryEngine,
-    metadata: {
-      name: "abramov_query_engine",
-      description: "A query engine for the Abramov documents",
-    },
-  });
-
-  // Create an OpenAIAgent with the function tools
-  const agent = new OpenAIAgent({
-    tools: [queryEngineTool],
-    verbose: true,
-  });
-
-  // Chat with the agent
-  const response = await agent.chat({
-    message: "What was his salary?",
-  });
-
-  // Print the response
-  console.log(String(response));
-}
-
-main().then(() => {
-  console.log("Done");
-});

examples/agent/query_openai_agent.ts

@@ -8,7 +8,7 @@ import {
 async function main() {
   // Load the documents
   const documents = await new SimpleDirectoryReader().loadData({
-    directoryPath: "node_modules/llamaindex/examples/",
+    directoryPath: "node_modules/llamaindex/examples",
   });
 
   // Create a vector index from the documents

examples/openai.ts

@@ -1,4 +1,4 @@
-import { OpenAI } from "llamaindex";
+import { OpenAI, OpenAIEmbedding } from "llamaindex";
 
 (async () => {
   const llm = new OpenAI({ model: "gpt-4-1106-preview", temperature: 0.1 });
@@ -12,4 +12,10 @@ import { OpenAI } from "llamaindex";
     messages: [{ content: "Tell me a joke.", role: "user" }],
   });
   console.log(response2.message.content);
+
+  // embeddings
+  const embedModel = new OpenAIEmbedding();
+  const texts = ["hello", "world"];
+  const embeddings = await embedModel.getTextEmbeddingsBatch(texts);
+  console.log(`\nWe have ${embeddings.length} embeddings`);
 })();

examples/pinecone-vector-store/README.md

@@ -7,8 +7,9 @@ There are two scripts available here: load-docs.ts and query.ts
 You'll need a Pinecone account, project, and index. Pinecone does not allow automatic creation of indexes on the free plan,
 so this vector store does not check and create the index (unlike, e.g., the PGVectorStore)
 
-Set the **PINECONE_API_KEY** and **PINECONE_ENVIRONMENT** environment variables to match your specific values. You will likely also need to set **PINECONE_INDEX_NAME**, unless your
-index is the default value "llama".
+Set the **PINECONE_API_KEY** and **PINECONE_ENVIRONMENT** environment variables to match your specific values.
+You will likely also need to set **PINECONE_INDEX_NAME**, unless your index is the default value "llama".
+By default, all operations take place inside the default namespace '', but you can set **PINECONE_NAMESPACE** to a different value if you need to.
 
 You'll also need a value for OPENAI_API_KEY in your environment.
 

packages/core/.gitignore

@@ -1 +1,3 @@
 .turbo
+README.md
+LICENSE

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # llamaindex
 
+## 0.1.19
+
+### Patch Changes
+
+- 026d068: feat: enhance pinecone usage
+
 ## 0.1.18
 
 ### Patch Changes

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.1.18",
+  "version": "0.1.19",
   "license": "MIT",
   "type": "module",
   "dependencies": {
@@ -95,7 +95,8 @@
     "build:esm": "swc src -d dist --strip-leading-paths --config-file .swcrc",
     "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file .cjs.swcrc",
     "build:type": "pnpm run -w type-check",
-    "postbuild": "node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
+    "copy": "cp -r ../../README.md ../../LICENSE .",
+    "postbuild": "pnpm run copy && node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
     "circular-check": "madge -c ./src/index.ts",
     "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\""
   }

packages/core/src/llm/ollama.ts

@@ -37,14 +37,18 @@ export class Ollama extends BaseEmbedding implements LLM {
   additionalChatOptions?: Record<string, unknown>;
   callbackManager?: CallbackManager;
 
+  protected modelMetadata: Partial<LLMMetadata>;
+
   constructor(
     init: Partial<Ollama> & {
       // model is required
       model: string;
+      modelMetadata?: Partial<LLMMetadata>;
     },
   ) {
     super();
     this.model = init.model;
+    this.modelMetadata = init.modelMetadata ?? {};
     Object.assign(this, init);
   }
 
@@ -56,6 +60,7 @@ export class Ollama extends BaseEmbedding implements LLM {
       maxTokens: undefined,
       contextWindow: this.contextWindow,
       tokenizer: undefined,
+      ...this.modelMetadata,
     };
   }
 

packages/core/src/storage/vectorStore/PGVectorStore.ts

@@ -1,5 +1,4 @@
-import pg from "pg";
-import pgvector from "pgvector/pg";
+import type pg from "pg";
 
 import type {
   VectorStore,
@@ -83,16 +82,18 @@ export class PGVectorStore implements VectorStore {
   private async getDb(): Promise<pg.Client> {
     if (!this.db) {
       try {
+        const { Client } = await import("pg");
+        const { registerType } = await import("pgvector/pg");
         // Create DB connection
         // Read connection params from env - see comment block above
-        const db = new pg.Client({
+        const db = new Client({
           connectionString: this.connectionString,
         });
         await db.connect();
 
         // Check vector extension
         db.query("CREATE EXTENSION IF NOT EXISTS vector");
-        await pgvector.registerType(db);
+        await registerType(db);
 
         // Check schema, table(s), index(es)
         await this.checkSchema(db);

packages/core/src/storage/vectorStore/PineconeVectorStore.ts

@@ -12,13 +12,15 @@ import type {
   Index,
   ScoredPineconeRecord,
 } from "@pinecone-database/pinecone";
-import { Pinecone } from "@pinecone-database/pinecone";
+import { type Pinecone } from "@pinecone-database/pinecone";
 import type { BaseNode, Metadata } from "../../Node.js";
 import { metadataDictToNode, nodeToMetadata } from "./utils.js";
 
 type PineconeParams = {
   indexName?: string;
   chunkSize?: number;
+  namespace?: string;
+  textKey?: string;
 };
 
 /**
@@ -37,18 +39,23 @@ export class PineconeVectorStore implements VectorStore {
   */
   db?: Pinecone;
   indexName: string;
+  namespace: string;
   chunkSize: number;
+  textKey: string;
 
   constructor(params?: PineconeParams) {
     this.indexName =
       params?.indexName ?? process.env.PINECONE_INDEX_NAME ?? "llama";
+    this.namespace = params?.namespace ?? process.env.PINECONE_NAMESPACE ?? "";
     this.chunkSize =
       params?.chunkSize ??
       Number.parseInt(process.env.PINECONE_CHUNK_SIZE ?? "100");
+    this.textKey = params?.textKey ?? "text";
   }
 
   private async getDb(): Promise<Pinecone> {
     if (!this.db) {
+      const { Pinecone } = await import("@pinecone-database/pinecone");
       this.db = await new Pinecone();
     }
 
@@ -148,24 +155,23 @@ export class PineconeVectorStore implements VectorStore {
     };
 
     const idx = await this.index();
-    const results = await idx.query(options);
+    const results = await idx.namespace(this.namespace).query(options);
 
     const idList = results.matches.map((row) => row.id);
-    const records: FetchResponse<any> = await idx.fetch(idList);
+    const records: FetchResponse<any> = await idx
+      .namespace(this.namespace)
+      .fetch(idList);
     const rows = Object.values(records.records);
 
     const nodes = rows.map((row) => {
-      const metadata = this.metaWithoutText(row.metadata);
-      const text = this.textFromResultRow(row);
-      const node = metadataDictToNode(metadata, {
+      const node = metadataDictToNode(row.metadata, {
         fallback: {
           id: row.id,
-          text,
-          metadata,
+          text: this.textFromResultRow(row),
+          metadata: this.metaWithoutText(row.metadata),
           embedding: row.values,
         },
       });
-      node.setContent(text);
       return node;
     });
 
@@ -199,12 +205,12 @@ export class PineconeVectorStore implements VectorStore {
   }
 
   textFromResultRow(row: ScoredPineconeRecord<Metadata>): string {
-    return row.metadata?.text ?? "";
+    return row.metadata?.[this.textKey] ?? "";
   }
 
   metaWithoutText(meta: Metadata): any {
     return Object.keys(meta)
-      .filter((key) => key != "text")
+      .filter((key) => key != this.textKey)
       .reduce((acc: any, key: string) => {
         acc[key] = meta[key];
         return acc;

packages/eslint-config-custom/index.js

@@ -36,6 +36,7 @@ module.exports = {
           "PINECONE_INDEX_NAME",
           "PINECONE_CHUNK_SIZE",
           "PINECONE_INDEX_NAME",
+          "PINECONE_NAMESPACE",
 
           "AZURE_OPENAI_API_KEY",
           "AZURE_OPENAI_API_INSTANCE_NAME",