diff --git a/backend/endpoints/v1/tools/index.js b/backend/endpoints/v1/tools/index.js
index a3e0a9f..30fffd6 100644
--- a/backend/endpoints/v1/tools/index.js
+++ b/backend/endpoints/v1/tools/index.js
@@ -1,8 +1,13 @@
+const { DocumentVectors } = require("../../../models/documentVectors");
const { Organization } = require("../../../models/organization");
const {
OrganizationConnection,
} = require("../../../models/organizationConnection");
+const {
+ OrganizationWorkspace,
+} = require("../../../models/organizationWorkspace");
const { Queue } = require("../../../models/queue");
+const { SystemSettings } = require("../../../models/systemSettings");
const {
userFromSession,
validSessionForUser,
@@ -14,6 +19,8 @@ const {
const {
organizationResetJob,
} = require("../../../utils/jobs/organizationResetJob");
+const { OpenAi } = require("../../../utils/openAi");
+const { selectConnector } = require("../../../utils/vectordatabases/providers");
process.env.NODE_ENV === "development"
? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` })
@@ -58,13 +65,11 @@ function toolEndpoints(app) {
organization_id: Number(organization.id),
});
if (!originalConnector) {
- response
- .status(200)
- .json({
- success: false,
- message:
- "No vector database is connected to the original organization.",
- });
+ response.status(200).json({
+ success: false,
+ message:
+ "No vector database is connected to the original organization.",
+ });
return;
}
@@ -83,13 +88,11 @@ function toolEndpoints(app) {
organization_id: Number(destinationOrg.id),
});
if (!destinationConnector) {
- response
- .status(200)
- .json({
- success: false,
- message:
- "No vector database is connected to the destination organization.",
- });
+ response.status(200).json({
+ success: false,
+ message:
+ "No vector database is connected to the destination organization.",
+ });
return;
}
@@ -149,12 +152,10 @@ function toolEndpoints(app) {
organization_id: Number(organization.id),
});
if (!connector) {
- response
- .status(200)
- .json({
- success: false,
- message: "No vector database is connected to this organization.",
- });
+ response.status(200).json({
+ success: false,
+ message: "No vector database is connected to this organization.",
+ });
return;
}
@@ -181,6 +182,112 @@ function toolEndpoints(app) {
}
}
);
+
+ app.post(
+ "/v1/tools/org/:orgSlug/workspace-similarity-search",
+ [validSessionForUser],
+ async function (request, response) {
+ try {
+ let queryVector;
+ const { orgSlug } = request.params;
+ const {
+ workspaceId,
+ input,
+ inputType = "text",
+ topK = 3,
+ } = reqBody(request);
+ const user = await userFromSession(request);
+ if (!user || user.role !== "admin") {
+ response.sendStatus(403).end();
+ return;
+ }
+
+ const organization = await Organization.getWithOwner(user.id, {
+ slug: orgSlug,
+ });
+ if (!organization) {
+ response.status(200).json({ results: [], error: "No org found." });
+ return;
+ }
+
+ const workspace = await OrganizationWorkspace.get({
+ id: workspaceId,
+ organization_id: organization.id,
+ });
+ if (!workspace) {
+ response
+ .status(200)
+ .json({ results: [], error: "No workspace found." });
+ return;
+ }
+
+ const connector = await OrganizationConnection.get({
+ organization_id: Number(organization.id),
+ });
+ if (!connector) {
+ response.status(200).json({
+ results: [],
+ error: "No vector database is connected to this organization.",
+ });
+ return;
+ }
+
+ if (inputType === "text") {
+ if (input?.length === 0) {
+ response.status(200).json({
+ results: [],
+ error: "No input data to embed.",
+ });
+ return;
+ }
+
+ const openAiKey = (
+ await SystemSettings.get({ label: "open_ai_api_key" })
+ )?.value;
+ if (!openAiKey) {
+ response.status(200).json({
+ results: [],
+ error: "No embedding API key set - cannot embed text data.",
+ });
+ return;
+ }
+
+ const openai = new OpenAi(openAiKey);
+ queryVector = await openai.embedTextChunk(input);
+ } else {
+ queryVector = input;
+ }
+
+ if (!queryVector || queryVector?.length === 0) {
+ response.status(200).json({
+ results: [],
+ error: "Failed to embed or parse input data.",
+ });
+ return;
+ }
+
+ const vectorDb = selectConnector(connector);
+ const searchResults = await vectorDb.similarityResponse(
+ workspace.fname,
+ queryVector,
+ topK
+ );
+ const results = searchResults.vectorIds.map((_, i) => {
+ return {
+ vectorId: searchResults.vectorIds[i],
+ text: searchResults.contextTexts[i],
+ metadata: searchResults.sourceDocuments[i],
+ score: searchResults.scores[i],
+ };
+ });
+
+ response.status(200).json({ results, error: null });
+ } catch (e) {
+ console.log(e.message, e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
}
module.exports = { toolEndpoints };
diff --git a/backend/utils/search/documentEmbeddings/semantic.js b/backend/utils/search/documentEmbeddings/semantic.js
index 197cbfe..604b54f 100644
--- a/backend/utils/search/documentEmbeddings/semantic.js
+++ b/backend/utils/search/documentEmbeddings/semantic.js
@@ -38,11 +38,11 @@ async function semanticSearch(document, query) {
// From similarity search we can find all document vector DB items to infer their associated
// document record.
- const searchString = searchResults.vectorIds
- .map((vid) => `'${vid}'`)
- .join(",");
const fragments = await DocumentVectors.where(
- { vectorId: { in: queryString }, document_id: Number(document.id) },
+ {
+ vectorId: { in: searchResults?.vectorIds || [] },
+ document_id: Number(document.id),
+ },
100
);
return { fragments, error: null };
diff --git a/backend/utils/search/workspaceDocuments/semantic.js b/backend/utils/search/workspaceDocuments/semantic.js
index 6417c64..2c60542 100644
--- a/backend/utils/search/workspaceDocuments/semantic.js
+++ b/backend/utils/search/workspaceDocuments/semantic.js
@@ -33,11 +33,8 @@ async function semanticSearch(workspace, query) {
// From similarity search we can find all document vector DB items to infer their associated
// document record.
- const searchString = searchResults.vectorIds
- .map((vid) => `'${vid}'`)
- .join(",");
const matchingDocumentVectors = await DocumentVectors.where({
- vectorId: { in: { searchString } },
+ vectorId: { in: searchResults?.vectorIds || [] },
});
const docDbIds = new Set();
matchingDocumentVectors.forEach((record) => docDbIds.add(record.document_id));
diff --git a/backend/utils/vectordatabases/providers/chroma/index.js b/backend/utils/vectordatabases/providers/chroma/index.js
index ec9c311..67bd455 100644
--- a/backend/utils/vectordatabases/providers/chroma/index.js
+++ b/backend/utils/vectordatabases/providers/chroma/index.js
@@ -18,6 +18,13 @@ class Chroma {
return { type, settings };
}
+ distanceToScore(distance = null) {
+ if (distance === null || typeof distance !== "number") return 0.0;
+ if (distance >= 1.0) return 1;
+ if (distance <= 0) return 0;
+ return 1 - distance;
+ }
+
async connect() {
const { ChromaClient } = require("chromadb");
const { type, settings } = this.config;
@@ -221,23 +228,25 @@ class Chroma {
}
}
- async similarityResponse(namespace, queryVector) {
+ async similarityResponse(namespace, queryVector, topK = 4) {
const { client } = await this.connect();
const collection = await client.getCollection({ name: namespace });
const result = {
vectorIds: [],
contextTexts: [],
sourceDocuments: [],
+ scores: [],
};
const response = await collection.query({
queryEmbeddings: queryVector,
- nResults: 4,
+ nResults: topK,
});
response.ids[0].forEach((_, i) => {
result.vectorIds.push(response.ids[0][i]);
result.contextTexts.push(response.documents[0][i]);
result.sourceDocuments.push(response.metadatas[0][i]);
+ result.scores.push(this.distanceToScore(response.distances[0][i]));
});
return result;
diff --git a/backend/utils/vectordatabases/providers/pinecone/index.js b/backend/utils/vectordatabases/providers/pinecone/index.js
index 1e63885..c23f9a8 100644
--- a/backend/utils/vectordatabases/providers/pinecone/index.js
+++ b/backend/utils/vectordatabases/providers/pinecone/index.js
@@ -374,18 +374,19 @@ class Pinecone {
}
}
- async similarityResponse(namespace, queryVector) {
+ async similarityResponse(namespace, queryVector, topK = 4) {
const { pineconeIndex } = await this.connect();
const result = {
vectorIds: [],
contextTexts: [],
sourceDocuments: [],
+ scores: [],
};
const response = await pineconeIndex.query({
queryRequest: {
namespace,
vector: queryVector,
- topK: 4,
+ topK,
includeMetadata: true,
},
});
@@ -394,6 +395,7 @@ class Pinecone {
result.vectorIds.push(match.id);
result.contextTexts.push(match.metadata.text);
result.sourceDocuments.push(match);
+ result.scores.push(match.score);
});
return result;
diff --git a/backend/utils/vectordatabases/providers/qdrant/index.js b/backend/utils/vectordatabases/providers/qdrant/index.js
index 10edb25..d23a8ce 100644
--- a/backend/utils/vectordatabases/providers/qdrant/index.js
+++ b/backend/utils/vectordatabases/providers/qdrant/index.js
@@ -195,17 +195,18 @@ class QDrant {
}
}
- async similarityResponse(namespace, queryVector) {
+ async similarityResponse(namespace, queryVector, topK = 4) {
const { client } = await this.connect();
const result = {
vectorIds: [],
contextTexts: [],
sourceDocuments: [],
+ scores: [],
};
const responses = await client.search(namespace, {
vector: queryVector,
- limit: 4,
+ limit: topK,
with_payload: true,
});
@@ -216,6 +217,7 @@ class QDrant {
id: response.id,
});
result.vectorIds.push(response.id);
+ result.scores.push(response.score);
});
return result;
diff --git a/backend/utils/vectordatabases/providers/weaviate/index.js b/backend/utils/vectordatabases/providers/weaviate/index.js
index 8dd21c1..ebd8591 100644
--- a/backend/utils/vectordatabases/providers/weaviate/index.js
+++ b/backend/utils/vectordatabases/providers/weaviate/index.js
@@ -379,28 +379,31 @@ class Weaviate {
}
}
- async similarityResponse(namespace, queryVector) {
+ async similarityResponse(namespace, queryVector, topK = 4) {
const { client } = await this.connect();
const className = this.camelCase(namespace);
const result = {
vectorIds: [],
contextTexts: [],
sourceDocuments: [],
+ scores: [],
};
const fieldsForCollection = await this.fieldNamesForCollection(namespace);
- const queryString = `${fieldsForCollection.join(" ")} _additional { id }`;
+ const queryString = `${fieldsForCollection.join(
+ " "
+ )} _additional { id certainty }`;
const queryResponse = await client.graphql
.get()
.withClassName(className)
.withFields(queryString)
.withNearVector({ vector: queryVector })
- .withLimit(4)
+ .withLimit(topK)
.do();
const responses = queryResponse?.data?.Get?.[className];
responses.forEach((response) => {
const {
- _additional: { id },
+ _additional: { id, certainty },
...rest
} = response;
result.contextTexts.push(rest?.text || "");
@@ -409,6 +412,7 @@ class Weaviate {
id,
});
result.vectorIds.push(id);
+ result.scores.push(certainty);
});
return result;
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 4f8597b..c1c3b0d 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -28,6 +28,7 @@ const MigrateConnectionView = lazy(
() => import('./pages/Tools/MigrateConnection')
);
const ResetConnectionView = lazy(() => import('./pages/Tools/ResetConnection'));
+const RAGDriftTestingView = lazy(() => import('./pages/Tools/RAGDrift'));
function App() {
return (
@@ -88,6 +89,10 @@ function App() {
path="/dashboard/:slug/tools/db-reset"
element={
below are a list of advanced {APP_NAME} only tools and services that - you can use to manage your vectorized data. + you can use to manage your connected vector database.