diff --git a/backend/endpoints/v1/index.js b/backend/endpoints/v1/index.js index cea2ebf..207de8e 100644 --- a/backend/endpoints/v1/index.js +++ b/backend/endpoints/v1/index.js @@ -6,6 +6,7 @@ const { documentProcessorEndpoints } = require("./document-processor"); const { documentEndpoints } = require("./documents"); const { jobEndpoints } = require("./jobs"); const { organizationEndpoints } = require("./organizations"); +const { toolEndpoints } = require("./tools"); const { userEndpoints } = require("./users"); const { workspaceEndpoints } = require("./workspaces"); @@ -36,6 +37,7 @@ function v1Endpoints(app) { documentEndpoints(app); documentProcessorEndpoints(app); jobEndpoints(app); + toolEndpoints(app); } module.exports = { v1Endpoints }; diff --git a/backend/endpoints/v1/organizations/index.js b/backend/endpoints/v1/organizations/index.js index aec31dd..fcc7d40 100644 --- a/backend/endpoints/v1/organizations/index.js +++ b/backend/endpoints/v1/organizations/index.js @@ -596,6 +596,7 @@ function organizationEndpoints(app) { documents: "countForEntity", vectors: "calcVectors", "cache-size": "calcVectorCache", + vectorCounts: "vectorCount", }; if (!Object.keys(methods).includes(statistic)) { diff --git a/backend/endpoints/v1/tools/index.js b/backend/endpoints/v1/tools/index.js new file mode 100644 index 0000000..a77380b --- /dev/null +++ b/backend/endpoints/v1/tools/index.js @@ -0,0 +1,82 @@ +const { Organization } = require("../../../models/organization"); +const { Queue } = require("../../../models/queue"); +const { + userFromSession, + validSessionForUser, + reqBody, +} = require("../../../utils/http"); +const { + organizationMigrationJob, +} = require("../../../utils/jobs/organizationMigrationJob"); + +process.env.NODE_ENV === "development" + ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` }) + : require("dotenv").config(); + +function toolEndpoints(app) { + if (!app) return; + + app.post( + "/v1/tools/org/:orgSlug/migrate", + [validSessionForUser], + async function (request, response) { + try { + const { orgSlug } = request.params; + const { destinationOrgId } = reqBody(request); + const user = await userFromSession(request); + if (!user || user.role !== "admin") { + response.sendStatus(403).end(); + return; + } + + const organization = await Organization.getWithOwner(user.id, { + slug: orgSlug, + }); + if (!organization) { + response + .status(200) + .json({ success: false, message: "No org found." }); + return; + } + + const destinationOrg = await Organization.getWithOwner(user.id, { + id: Number(destinationOrgId), + }); + if (!destinationOrg) { + response + .status(200) + .json({ + success: false, + message: "Destination org does not exit.", + }); + return; + } + + const existingJobForOrg = await Queue.get({ + taskName: "workspace/migrate", + status: "pending", + organization_id: organization.id, + }); + const existingJobForDestinationOrg = await Queue.get({ + taskName: "workspace/migrate", + status: "pending", + organization_id: destinationOrg.id, + }); + //TODO - uncomment if (!!existingJobForOrg || !!existingJobForDestinationOrg) { + // response.status(200).json({ success: false, message: "There is an existing migration job already running for these organizations." }); + // return; + // } + + await organizationMigrationJob(organization, destinationOrg, user); + response + .status(200) + .json({ success: true, message: "Migration job queued." }); + } catch (e) { + console.log(e.message, e); + response.sendStatus(500).end(); + } + } + ); +} + +module.exports = { toolEndpoints }; diff --git a/backend/models/workspaceDocument.js b/backend/models/workspaceDocument.js index 4a6622f..a685234 100644 --- a/backend/models/workspaceDocument.js +++ b/backend/models/workspaceDocument.js @@ -3,6 +3,7 @@ const path = require("path"); const { v5 } = require("uuid"); const { fetchMetadata } = require("../utils/storage"); const { DocumentVectors } = require("./documentVectors"); +const { selectConnector } = require("../utils/vectordatabases/providers"); const WorkspaceDocument = { vectorFilenameRaw: function (documentName, workspaceId) { @@ -158,6 +159,25 @@ const WorkspaceDocument = { return totalBytes; }, + // Will get both the remote and local count of vectors to see if the numbers match. + vectorCount: async function (field = "organization_id", value = null) { + try { + const { OrganizationConnection } = require("./organizationConnection"); + const connector = await OrganizationConnection.get({ [field]: value }); + if (!connector) return { remoteCount: 0, localCount: 0 }; + const vectorDb = selectConnector(connector); + + return { + remoteCount: (await vectorDb.totalIndicies())?.result, + localCount: await DocumentVectors.count({ + [field]: value, + }), + }; + } catch (e) { + console.error(e); + return 0; + } + }, }; module.exports.WorkspaceDocument = WorkspaceDocument; diff --git a/backend/utils/jobs/organizationMigrationJob/index.js b/backend/utils/jobs/organizationMigrationJob/index.js new file mode 100644 index 0000000..1f0f89f --- /dev/null +++ b/backend/utils/jobs/organizationMigrationJob/index.js @@ -0,0 +1,29 @@ +const { Queue } = require("../../../models/queue"); + +async function organizationMigrationJob( + organization, + destinationOrganization, + user +) { + const taskName = `organization/migrate`; + const jobData = { organization, destinationOrganization }; + const { job, error } = await Queue.create( + taskName, + jobData, + user.id, + organization.id + ); + if (!!error) return { job, error }; + await Queue.sendJob({ + name: taskName, + data: { + jobId: job.id, + ...jobData, + }, + }); + return { job, error: null }; +} + +module.exports = { + organizationMigrationJob, +}; diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b8695de..5d648ab 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -24,6 +24,9 @@ const OnboardingSecuritySetup = lazy( const OrganizationJobsView = lazy(() => import('./pages/Jobs')); const OrganizationToolsView = lazy(() => import('./pages/Tools')); const SystemSettingsView = lazy(() => import('./pages/SystemSettings')); +const MigrateConnectionView = lazy( + () => import('./pages/Tools/MigrateConnection') +); function App() { return ( @@ -78,7 +81,7 @@ function App() { } + element={} /> } /> diff --git a/frontend/src/components/Sidebar/index.tsx b/frontend/src/components/Sidebar/index.tsx index b2a117d..ed4fe80 100644 --- a/frontend/src/components/Sidebar/index.tsx +++ b/frontend/src/components/Sidebar/index.tsx @@ -95,8 +95,9 @@ export default function Sidebar({ <>