diff --git a/backend/endpoints/v1/tools/index.js b/backend/endpoints/v1/tools/index.js index 382b425..74e52fa 100644 --- a/backend/endpoints/v1/tools/index.js +++ b/backend/endpoints/v1/tools/index.js @@ -1,10 +1,8 @@ -const { DocumentVectors } = require("../../../models/documentVectors"); const { Organization } = require("../../../models/organization"); const { OrganizationConnection, } = require("../../../models/organizationConnection"); const { Queue } = require("../../../models/queue"); -const { RagTest } = require("../../../models/ragTest"); const { userFromSession, validSessionForUser, @@ -16,10 +14,10 @@ const { const { organizationResetJob, } = require("../../../utils/jobs/organizationResetJob"); -const { createRagTest } = require("../../../utils/toolHelpers/RagTests/create"); const { workspaceSimilaritySearch, } = require("../../../utils/toolHelpers/workspaceSimilaritySearch"); +const { ragTestingEndpoints } = require("./ragTesting"); process.env.NODE_ENV === "development" ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` }) @@ -27,6 +25,7 @@ process.env.NODE_ENV === "development" function toolEndpoints(app) { if (!app) return; + ragTestingEndpoints(app); app.post( "/v1/tools/org/:orgSlug/migrate", @@ -182,150 +181,6 @@ function toolEndpoints(app) { } ); - app.get( - "/v1/tools/org/:orgSlug/rag-tests", - [validSessionForUser], - async function (request, response) { - try { - const { orgSlug } = request.params; - const user = await userFromSession(request); - if (!user || user.role !== "admin") { - response.sendStatus(403).end(); - return; - } - - const organization = await Organization.getWithOwner(user.id, { - slug: orgSlug, - }); - if (!organization) { - response.status(200).json({ ragTests: [], message: "No org found." }); - return; - } - - const tests = await RagTest.where( - { - organization_id: organization.id, - }, - null, - { lastRun: "desc" }, - { - id: true, - promptText: true, - frequencyType: true, - topK: true, - lastRun: true, - comparisons: true, - promptVector: true, - workspace: true, - organization: true, - organization_rag_test_runs: { - select: { - id: true, - status: true, - }, - orderBy: { - id: "desc", - }, - }, - } - ); - response.status(200).json({ ragTests: tests, message: null }); - return; - } catch (e) { - console.log(e.message, e); - response.sendStatus(500).end(); - } - } - ); - - app.get( - "/v1/tools/org/:orgSlug/rag-tests/:testId", - [validSessionForUser], - async function (request, response) { - try { - const { orgSlug, testId } = request.params; - const user = await userFromSession(request); - if (!user || user.role !== "admin") { - response.sendStatus(403).end(); - return; - } - - const organization = await Organization.getWithOwner(user.id, { - slug: orgSlug, - }); - if (!organization) { - response.status(200).json({ test: null, message: "No org found." }); - return; - } - - const test = await RagTest.get({ id: Number(testId) }); - const runs = await RagTest.getRuns(test.id, {}, 10, { - createdAt: "desc", - }); - response.status(200).json({ test, runs, message: null }); - return; - } catch (e) { - console.log(e.message, e); - response.sendStatus(500).end(); - } - } - ); - - app.delete( - "/v1/tools/org/:orgSlug/rag-tests/:testId", - [validSessionForUser], - async function (request, response) { - try { - const { orgSlug, testId } = request.params; - const user = await userFromSession(request); - if (!user || user.role !== "admin") { - response.sendStatus(403).end(); - return; - } - - const organization = await Organization.getWithOwner(user.id, { - slug: orgSlug, - }); - if (!organization) { - response.sendStatus(400).end(); - return; - } - - const test = await RagTest.get({ id: Number(testId) }, { id: true }); - if (!test) { - response.sendStatus(400).end(); - return; - } - - await RagTest.delete({ id: test.id }); - response.sendStatus(200).end(); - return; - } catch (e) { - console.log(e.message, e); - response.sendStatus(500).end(); - } - } - ); - - app.post( - "/v1/tools/org/:orgSlug/rag-tests/create", - [validSessionForUser], - async function (request, response) { - try { - const user = await userFromSession(request); - if (!user || user.role !== "admin") { - response.sendStatus(403).end(); - return; - } - - return await createRagTest(user, request, response); - } catch (e) { - console.log(e.message, e); - response.sendStatus(500).end(); - } - } - ); - app.post( "/v1/tools/org/:orgSlug/workspace-similarity-search", [validSessionForUser], diff --git a/backend/endpoints/v1/tools/ragTesting/index.js b/backend/endpoints/v1/tools/ragTesting/index.js new file mode 100644 index 0000000..70afba6 --- /dev/null +++ b/backend/endpoints/v1/tools/ragTesting/index.js @@ -0,0 +1,213 @@ +const { Organization } = require("../../../../models/organization"); +const { Queue } = require("../../../../models/queue"); +const { RagTest } = require("../../../../models/ragTest"); +const { + userFromSession, + validSessionForUser, +} = require("../../../../utils/http"); +const { + createRagTestJobRun, +} = require("../../../../utils/jobs/createRagTestJobRun"); +const { + createRagTest, +} = require("../../../../utils/toolHelpers/RagTests/create"); +const { + workspaceSimilaritySearch, +} = require("../../../../utils/toolHelpers/workspaceSimilaritySearch"); + +process.env.NODE_ENV === "development" + ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` }) + : require("dotenv").config(); + +function ragTestingEndpoints(app) { + if (!app) return; + + app.get( + "/v1/tools/org/:orgSlug/rag-tests", + [validSessionForUser], + async function (request, response) { + try { + const { orgSlug } = request.params; + const user = await userFromSession(request); + if (!user || user.role !== "admin") { + response.sendStatus(403).end(); + return; + } + + const organization = await Organization.getWithOwner(user.id, { + slug: orgSlug, + }); + if (!organization) { + response.status(200).json({ ragTests: [], message: "No org found." }); + return; + } + + const tests = await RagTest.where( + { + organization_id: organization.id, + }, + null, + { lastRun: "desc" }, + { + id: true, + promptText: true, + frequencyType: true, + topK: true, + lastRun: true, + comparisons: true, + promptVector: true, + workspace: true, + organization: true, + organization_rag_test_runs: { + select: { + id: true, + status: true, + }, + orderBy: { + id: "desc", + }, + }, + } + ); + response.status(200).json({ ragTests: tests, message: null }); + return; + } catch (e) { + console.log(e.message, e); + response.sendStatus(500).end(); + } + } + ); + + app.get( + "/v1/tools/org/:orgSlug/rag-tests/:testId", + [validSessionForUser], + async function (request, response) { + try { + const { orgSlug, testId } = request.params; + const user = await userFromSession(request); + if (!user || user.role !== "admin") { + response.sendStatus(403).end(); + return; + } + + const organization = await Organization.getWithOwner(user.id, { + slug: orgSlug, + }); + if (!organization) { + response.status(200).json({ test: null, message: "No org found." }); + return; + } + + const test = await RagTest.get({ id: Number(testId) }); + const runs = await RagTest.getRuns(test.id, {}, 10, { + createdAt: "desc", + }); + response.status(200).json({ test, runs, message: null }); + return; + } catch (e) { + console.log(e.message, e); + response.sendStatus(500).end(); + } + } + ); + + app.delete( + "/v1/tools/org/:orgSlug/rag-tests/:testId", + [validSessionForUser], + async function (request, response) { + try { + const { orgSlug, testId } = request.params; + const user = await userFromSession(request); + if (!user || user.role !== "admin") { + response.sendStatus(403).end(); + return; + } + + const organization = await Organization.getWithOwner(user.id, { + slug: orgSlug, + }); + if (!organization) { + response.sendStatus(400).end(); + return; + } + + const test = await RagTest.get({ id: Number(testId) }, { id: true }); + if (!test) { + response.sendStatus(400).end(); + return; + } + + await RagTest.delete({ id: test.id }); + response.sendStatus(200).end(); + return; + } catch (e) { + console.log(e.message, e); + response.sendStatus(500).end(); + } + } + ); + + app.post( + "/v1/tools/org/:orgSlug/rag-tests/create", + [validSessionForUser], + async function (request, response) { + try { + const user = await userFromSession(request); + if (!user || user.role !== "admin") { + response.sendStatus(403).end(); + return; + } + + return await createRagTest(user, request, response); + } catch (e) { + console.log(e.message, e); + response.sendStatus(500).end(); + } + } + ); + + app.post( + "/v1/tools/org/:orgSlug/rag-tests/:testId/run", + [validSessionForUser], + async function (request, response) { + try { + const { orgSlug, testId } = request.params; + const user = await userFromSession(request); + if (!user || user.role !== "admin") { + response.sendStatus(403).end(); + return; + } + + const organization = await Organization.getWithOwner(user.id, { + slug: orgSlug, + }); + if (!organization) { + response + .status(200) + .json({ success: false, error: "No organization found." }); + return; + } + + const test = await RagTest.get({ id: Number(testId) }, { id: true }); + if (!test) { + response + .status(200) + .json({ success: false, error: "No test found for that id." }); + return; + } + + const { job, error } = await createRagTestJobRun( + organization, + test.id, + user + ); + response.status(200).json({ job, error }); + } catch (e) { + console.log(e.message, e); + response.sendStatus(500).end(); + } + } + ); +} + +module.exports = { ragTestingEndpoints }; diff --git a/backend/utils/jobs/createRagTestJobRun/index.js b/backend/utils/jobs/createRagTestJobRun/index.js new file mode 100644 index 0000000..620b829 --- /dev/null +++ b/backend/utils/jobs/createRagTestJobRun/index.js @@ -0,0 +1,38 @@ +const { Queue } = require("../../../models/queue"); + +async function createRagTestJobRun(organization, testId, user) { + const taskName = `rag-test/run`; + const pendingJob = await Queue.get({ + organization_id: Number(organization.id), + status: Queue.status.pending, + taskName, + }); + + if (pendingJob) { + const pendingJobData = JSON.parse(pendingJob.data); + if (pendingJobData.testId === testId) { + return { job: null, error: "A job like this is currently running." }; + } + } + + const jobData = { organization, testId }; + const { job, error } = await Queue.create( + taskName, + jobData, + user.id, + organization.id + ); + if (!!error) return { job, error }; + await Queue.sendJob({ + name: taskName, + data: { + jobId: job.id, + ...jobData, + }, + }); + return { job, error: null }; +} + +module.exports = { + createRagTestJobRun, +}; diff --git a/frontend/src/models/tools.ts b/frontend/src/models/tools.ts index d9ece75..1eccdd7 100644 --- a/frontend/src/models/tools.ts +++ b/frontend/src/models/tools.ts @@ -66,6 +66,23 @@ const Tools = { return { test: null, error: e.message }; }); }, + runRagTest: async ( + test: IRagTest + ): Promise<{ job: object | null; error: null | string }> => { + return fetch( + `${API_BASE}/v1/tools/org/${test.organization.slug}/rag-tests/${test.id}/run`, + { + method: 'POST', + cache: 'no-cache', + headers: baseHeaders(), + } + ) + .then((res) => res.json()) + .catch((e) => { + console.error(e); + return { job: null, error: e.message }; + }); + }, deleteRagTest: async (test: IRagTest): Promise => { return fetch( `${API_BASE}/v1/tools/org/${test.organization.slug}/rag-tests/${test.id}`, diff --git a/frontend/src/pages/Tools/RAGTesting/RecentTests/index.tsx b/frontend/src/pages/Tools/RAGTesting/RecentTests/index.tsx index 18e6e2e..d13555e 100644 --- a/frontend/src/pages/Tools/RAGTesting/RecentTests/index.tsx +++ b/frontend/src/pages/Tools/RAGTesting/RecentTests/index.tsx @@ -1,9 +1,10 @@ -import { Box, Trash } from 'react-feather'; +import { Box, Loader, Trash } from 'react-feather'; import Tools, { IRagTest } from '../../../../models/tools'; import moment from 'moment'; import paths from '../../../../utils/paths'; import showToast from '../../../../utils/toast'; import TestDetailsModal from '../TestDetails'; +import { useState } from 'react'; export default function RecentTestRuns({ tests, @@ -127,9 +128,7 @@ function TestItem({ test, onDelete }: { test: IRagTest; onDelete: any }) { View Runs )} - + + ); +} + function TestResultBadge({ test }: { test: IRagTest }) { const { organization_rag_test_runs = [] } = test; if (organization_rag_test_runs.length === 0) {