vector-admin/workers/functions/deleteDocument/index.js

const { Queue } = require('../../../backend/models/queue');
const {
  Chroma,
} = require('../../../backend/utils/vectordatabases/providers/chroma');
const { InngestClient } = require('../../utils/inngest');
const {
  WorkspaceDocument,
} = require('../../../backend/models/workspaceDocument');
const { DocumentVectors } = require('../../../backend/models/documentVectors');
const { deleteVectorCacheFile } = require('../../../backend/utils/storage');
const {
  Pinecone,
} = require('../../../backend/utils/vectordatabases/providers/pinecone');
const {
  QDrant,
} = require('../../../backend/utils/vectordatabases/providers/qdrant');
const {
  Weaviate,
} = require('../../../backend/utils/vectordatabases/providers/weaviate');

const deleteChromaDocument = InngestClient.createFunction(
  { name: 'Delete Document From ChromaDB' },
  { event: 'chroma/deleteDocument' },
  async ({ event, step: _step, logger }) => {
    var result = {};
    const { workspace, document, connector, jobId } = event.data;
    try {
      const chromaClient = new Chroma(connector);
      const { client } = await chromaClient.connect();
      const collection = await client.getCollection({ name: workspace.fname });

      const vectors = await DocumentVectors.where({
        document_id: Number(document.id),
      });
      const vectorIds = vectors.map((vector) => vector.vectorId);
      await collection.delete({ ids: vectorIds });
      await WorkspaceDocument.delete({ id: Number(document.id) });
      await deleteVectorCacheFile(WorkspaceDocument.vectorFilename(document));

      result = {
        message: `Document ${document.name} removed from Chroma collection ${workspace.name}.`,
      };
      await Queue.updateJob(jobId, Queue.status.complete, result);
      return { result };
    } catch (e) {
      const result = {
        message: `Job failed with error`,
        error: e.message,
        details: e,
      };
      await Queue.updateJob(jobId, Queue.status.failed, result);
      return { result };
    }
  }
);

const deletePineconeDocument = InngestClient.createFunction(
  { name: 'Delete Document From PineconeDB' },
  { event: 'pinecone/deleteDocument' },
  async ({ event, step: _step, logger }) => {
    var result = {};
    const { workspace, document, connector, jobId } = event.data;
    try {
      const pineconeClient = new Pinecone(connector);
      const { pineconeIndex } = await pineconeClient.connect();
      const hasNamespace = await pineconeClient.namespaceExists(
        pineconeIndex,
        workspace.fname
      );

      if (!hasNamespace) {
        result = {
          message: `No namespace found with name ${workspace.fname} - nothing to do.`,
        };
        await Queue.updateJob(jobId, Queue.status.failed, result);
        return { result };
      }

      const vectors = await DocumentVectors.where({
        document_id: Number(document.id),
      });
      const vectorIds = vectors.map((vector) => vector.vectorId);

      await pineconeIndex.delete1({
        ids: vectorIds,
        namespace: workspace.fname,
      });
      await WorkspaceDocument.delete({ id: Number(document.id) });
      await deleteVectorCacheFile(WorkspaceDocument.vectorFilename(document));

      result = {
        message: `Document ${document.name} removed from ${connector.type} collection ${workspace.name}.`,
      };
      await Queue.updateJob(jobId, Queue.status.complete, result);
      return { result };
    } catch (e) {
      const result = {
        message: `Job failed with error`,
        error: e.message,
        details: e,
      };
      await Queue.updateJob(jobId, Queue.status.failed, result);
      return { result };
    }
  }
);

const deleteQdrantDocument = InngestClient.createFunction(
  { name: 'Delete Document From QDrant' },
  { event: 'qdrant/deleteDocument' },
  async ({ event, step: _step, logger }) => {
    var result = {};
    const { workspace, document, connector, jobId } = event.data;
    try {
      const qdrantClient = new QDrant(connector);
      const { client } = await qdrantClient.connect();
      const hasNamespace = await qdrantClient.namespaceExists(
        client,
        workspace.fname
      );

      if (!hasNamespace) {
        result = {
          message: `No namespace found with name ${workspace.fname} - nothing to do.`,
        };
        await Queue.updateJob(jobId, Queue.status.failed, result);
        return { result };
      }

      const vectors = await DocumentVectors.where({
        document_id: Number(document.id),
      });
      const vectorIds = vectors.map((vector) => vector.vectorId);
      await client.delete(workspace.fname, {
        wait: true,
        points: vectorIds,
      });

      await WorkspaceDocument.delete({ id: Number(document.id) });
      await deleteVectorCacheFile(WorkspaceDocument.vectorFilename(document));

      result = {
        message: `Document ${document.name} removed from ${connector.type} collection ${workspace.name}.`,
      };
      await Queue.updateJob(jobId, Queue.status.complete, result);
      return { result };
    } catch (e) {
      const result = {
        message: `Job failed with error`,
        error: e.message,
        details: e,
      };
      await Queue.updateJob(jobId, Queue.status.failed, result);
      return { result };
    }
  }
);

const deleteWeaviateDocument = InngestClient.createFunction(
  { name: 'Delete Document From Weaviate' },
  { event: 'weaviate/deleteDocument' },
  async ({ event, step: _step, logger }) => {
    var result = {};
    const { workspace, document, connector, jobId } = event.data;
    try {
      const weaviateClient = new Weaviate(connector);
      const { client } = await weaviateClient.connect();
      const className = weaviateClient.camelCase(workspace.fname);
      const hasNamespace = await weaviateClient.namespaceExists(
        client,
        workspace.fname
      );

      if (!hasNamespace) {
        result = {
          message: `No namespace found with name ${workspace.fname} (class: ${className}) - nothing to do.`,
        };
        await Queue.updateJob(jobId, Queue.status.failed, result);
        return { result };
      }

      const vectors = await DocumentVectors.where({
        document_id: Number(document.id),
      });
      const vectorIds = vectors.map((vector) => vector.vectorId);

      await weaviateClient.bulkDeleteIds(client, className, vectorIds);
      await WorkspaceDocument.delete({ id: Number(document.id) });
      await deleteVectorCacheFile(WorkspaceDocument.vectorFilename(document));

      result = {
        message: `Document ${document.name} removed from ${connector.type} collection ${workspace.name}.`,
      };
      await Queue.updateJob(jobId, Queue.status.complete, result);
      return { result };
    } catch (e) {
      const result = {
        message: `Job failed with error`,
        error: e.message,
        details: e,
      };
      await Queue.updateJob(jobId, Queue.status.failed, result);
      return { result };
    }
  }
);

module.exports = {
  deleteChromaDocument,
  deletePineconeDocument,
  deleteQdrantDocument,
  deleteWeaviateDocument,
};