WIP for PG migration

backend/endpoints/auth.js

@@ -24,7 +24,7 @@ function authenticationEndpoints(app) {
       }
 
       if (email === process.env.SYS_EMAIL) {
-        const completeSetup = (await User.count('role = "admin"')) > 0;
+        const completeSetup = (await User.count({ role: "admin" })) > 0;
         if (completeSetup) {
           response.status(200).json({
             user: null,
@@ -36,7 +36,7 @@ function authenticationEndpoints(app) {
         }
       }
 
-      const existingUser = await User.get(`email = '${email}'`);
+      const existingUser = await User.get({ email: email });
       if (!existingUser) {
         response.status(200).json({
           user: null,
@@ -87,7 +87,7 @@ function authenticationEndpoints(app) {
         return;
       }
 
-      const adminCount = await User.count('role = "admin"');
+      const adminCount = await User.count({ role: "admin" });
       if (adminCount === 0) {
         response.status(200).json({
           user: null,
@@ -99,7 +99,7 @@ function authenticationEndpoints(app) {
         return;
       }
 
-      const existingUser = await User.get(`email = '${email}'`);
+      const existingUser = await User.get({ email });
       if (!!existingUser) {
         response.status(200).json({
           user: null,
@@ -110,9 +110,9 @@ function authenticationEndpoints(app) {
         return;
       }
 
-      const allowingAccounts = await SystemSettings.get(
-        'label = "allow_account_creation"'
-      );
+      const allowingAccounts = await SystemSettings.get({
+        label: "allow_account_creation",
+      });
       if (
         !!allowingAccounts &&
         allowingAccounts.value !== null &&
@@ -127,9 +127,9 @@ function authenticationEndpoints(app) {
         return;
       }
 
-      const domainRestriction = await SystemSettings.get(
-        'label = "account_creation_domain_scope"'
-      );
+      const domainRestriction = await SystemSettings.get({
+        label: "account_creation_domain_scope",
+      });
       if (
         !!domainRestriction &&
         domainRestriction.value !== null &&
@@ -155,7 +155,7 @@ function authenticationEndpoints(app) {
         return;
       }
 
-      await User.addToAllOrgs(user.id);
+      await User.addToAllOrgs(user.id); // TODO: Add users to Orgs
       await Telemetry.sendTelemetry("login_event");
       response.status(200).json({
         user,
@@ -183,7 +183,7 @@ function authenticationEndpoints(app) {
         return;
       }
 
-      const adminCount = await User.count('role = "admin"');
+      const adminCount = await User.count({ role: "admin" });
       if (adminCount > 0) {
         response.status(200).json({
           user: null,
@@ -195,7 +195,7 @@ function authenticationEndpoints(app) {
         return;
       }
 
-      const existingUser = await User.get(`email = '${email}'`);
+      const existingUser = await User.get({ email });
       if (!!existingUser) {
         response.status(200).json({
           user: null,

backend/endpoints/system.js

@@ -53,7 +53,7 @@ function systemEndpoints(app) {
           return;
         }
 
-        const config = await SystemSettings.get(`label = '${label}'`);
+        const config = await SystemSettings.get({ label });
         response.status(200).json({ label, exists: !!config?.value });
       } catch (e) {
         console.log(e.message, e);
@@ -79,7 +79,7 @@ function systemEndpoints(app) {
           return;
         }
 
-        const config = await SystemSettings.get(`label = '${label}'`);
+        const config = await SystemSettings.get({ label });
         if (SystemSettings.privateField.includes(label)) {
           response.status(200).json({
             ...config,

backend/endpoints/v1/organizations/index.js

@@ -552,8 +552,10 @@ function organizationEndpoints(app) {
           "ORDER BY createdAt DESC"
         );
         for (const job of jobs) {
-          const { id, email, role } = await User.get(`id = ${job.runByUserId}`);
-          job.runByUser = { id, email, role };
+          const { id, email, role } = await User.get({
+            id: Number(job.run_by_user_id),
+          });
+          job.run_by_user_id = { id, email, role };
         }
         response.status(200).json({ jobs });
       } catch (e) {

backend/endpoints/v1/users/index.js

@@ -23,7 +23,7 @@ function userEndpoints(app) {
           return;
         }
 
-        const users = await User.whereWithOrgs(`role != 'root'`);
+        const users = await User.whereWithOrgs(`role != 'root'`); // TODO: Support
         response.status(200).json({ users });
       } catch (e) {
         console.log(e.message, e);
@@ -51,7 +51,7 @@ function userEndpoints(app) {
           return;
         }
 
-        await User.delete(`id = ${userId}`);
+        await User.delete({ id: Number(userId) });
         response.status(200).json({ success: true, error: null });
       } catch (e) {
         console.log(e.message, e);
@@ -77,7 +77,7 @@ function userEndpoints(app) {
           password,
           role,
         });
-        await User.addToAllOrgs(newUser.id);
+        await User.addToAllOrgs(newUser.id); // TODO
         response.status(200).json({ success: !!newUser, error: message });
       } catch (e) {
         console.log(e.message, e);

backend/index.js

@@ -56,7 +56,7 @@ app
   .listen(process.env.SERVER_PORT || 3001, async () => {
     // await validateTablePragmas();
     await systemInit();
-    setupDebugger(apiRouter);
+    // setupDebugger(apiRouter); //TODO: DEBUGGER
     console.log(
       `Example app listening on port ${process.env.SERVER_PORT || 3001}`
     );

backend/models/systemSettings.js

@@ -1,4 +1,4 @@
-// const { checkForMigrations } = require("../utils/database");
+const prisma = require("../utils/prisma");
 
 const SystemSettings = {
   supportedFields: [
@@ -10,102 +10,52 @@ const SystemSettings = {
     "telemetry_id",
   ],
   privateField: ["open_ai_api_key"],
-  tablename: "system_settings",
-  colsInit: `
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  label TEXT UNIQUE NOT NULL,
-  value TEXT,
-  createdAt TEXT DEFAULT (strftime('%s', 'now')),
-  lastUpdatedAt TEXT DEFAULT (strftime('%s', 'now'))
-  `,
-  // migrateTable: async function () {
-  //   console.log(`\x1b[34m[MIGRATING]\x1b[0m Checking for Document migrations`);
-  //   const db = await this.db(false);
-  //   await checkForMigrations(this, db);
-  // },
-  migrations: function () {
-    return [];
+  get: async function (clause = {}) {
+    try {
+      const setting = await prisma.system_settings.findFirst({ where: clause });
+      return setting || null;
+    } catch (error) {
+      console.error(error.message);
+      return null;
+    }
   },
-  db: async function (tracing = true) {
-    const sqlite3 = require("sqlite3").verbose();
-    const { open } = require("sqlite");
-    const path = require("path");
-    const dbFilePath = path.resolve(__dirname, "../storage/vdbms.db");
-    const db = await open({
-      filename: dbFilePath,
-      driver: sqlite3.Database,
-    });
 
-    await db.exec(
-      `PRAGMA foreign_keys = ON;CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
-    );
-
-    if (tracing) db.on("trace", (sql) => console.log(sql));
-    return db;
+  where: async function (clause = {}, limit = null) {
+    try {
+      const settings = await prisma.system_settings.findMany({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+      });
+      return settings;
+    } catch (error) {
+      console.error(error.message);
+      return [];
+    }
   },
-  get: async function (clause = "") {
-    const db = await this.db();
-    const result = await db
-      .get(`SELECT * FROM ${this.tablename} WHERE ${clause}`)
-      .then((res) => res || null);
-    if (!result) return null;
-    await db.close();
 
-    return result;
-  },
-  where: async function (clause = null, limit = null) {
-    const db = await this.db();
-    const results = await db.all(
-      `SELECT * FROM ${this.tablename} ${clause ? `WHERE ${clause}` : ""} ${
-        !!limit ? `LIMIT ${limit}` : ""
-      }`
-    );
-    await db.close();
-
-    return results;
-  },
   updateSettings: async function (updates = {}) {
     const validConfigKeys = Object.keys(updates).filter((key) =>
       this.supportedFields.includes(key)
     );
+
     for (const key of validConfigKeys) {
-      const existingRecord = await this.get(`label = '${key}'`);
+      const existingRecord = await this.get({ label: key });
+
       if (!existingRecord) {
-        const db = await this.db();
         const value = updates[key] === null ? null : String(updates[key]);
-        const { success, message } = await db
-          .run(`INSERT INTO ${this.tablename} (label, value) VALUES (?, ?)`, [
-            key,
-            value,
-          ])
-          .then((res) => {
-            return { id: res.lastID, success: true, message: null };
-          })
-          .catch((error) => {
-            return { id: null, success: false, message: error.message };
-          });
-        await db.close();
+        const success = await prisma.system_settings.create({
+          data: { label: key, value },
+        });
         if (!success) {
           console.error("FAILED TO ADD SYSTEM CONFIG OPTION", message);
           return { success: false, error: message };
         }
       } else {
-        const db = await this.db();
         const value = updates[key] === null ? null : String(updates[key]);
-        const { success, message } = await db
-          .run(`UPDATE ${this.tablename} SET label=?,value=? WHERE id = ?`, [
-            key,
-            value,
-            existingRecord.id,
-          ])
-          .then(() => {
-            return { success: true, message: null };
-          })
-          .catch((error) => {
-            return { success: false, message: error.message };
-          });
-
-        await db.close();
+        const success = await prisma.system_settings.update({
+          where: { id: Number(existingRecord.id) },
+          data: { label: key, value },
+        });
         if (!success) {
           console.error("FAILED TO UPDATE SYSTEM CONFIG OPTION", message);
           return { success: false, error: message };

backend/models/telemetry.js

@@ -7,7 +7,7 @@ const Telemetry = {
   stubDevelopmentEvents: true, // [DO NOT TOUCH] Core team only.
   label: "telemetry_id",
   id: async function () {
-    const result = await SystemSettings.get(`label = '${this.label}'`);
+    const result = await SystemSettings.get({ label: this.label });
     if (!!result?.value) return result.value;
     return result?.value;
   },

backend/models/user.js

@@ -1,201 +1,147 @@
-// const { checkForMigrations } = require("../utils/database");
+const prisma = require("../utils/prisma");
 
 const User = {
-  tablename: "users",
-  colsInit: `
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  email TEXT UNIQUE,
-  password TEXT NOT NULL,
-  role TEXT NOT NULL DEFAULT "default",
-  createdAt TEXT DEFAULT (strftime('%s', 'now')),
-  lastUpdatedAt TEXT DEFAULT (strftime('%s', 'now'))
-  `,
-  // migrateTable: async function () {
-  //   console.log(`\x1b[34m[MIGRATING]\x1b[0m Checking for Document migrations`);
-  //   const db = await this.db(false);
-  //   await checkForMigrations(this, db);
-  // },
-  migrations: function () {
-    return [];
-  },
-  db: async function (tracing = true) {
-    const sqlite3 = require("sqlite3").verbose();
-    const { open } = require("sqlite");
-    const path = require("path");
-    const dbFilePath = path.resolve(__dirname, "../storage/vdbms.db");
-    const db = await open({
-      filename: dbFilePath,
-      driver: sqlite3.Database,
-    });
-
-    await db.exec(
-      `PRAGMA foreign_keys = ON; CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
-    );
-
-    if (tracing) db.on("trace", (sql) => console.log(sql));
-    return db;
-  },
   create: async function ({ email, password, role = null }) {
-    const bcrypt = require("bcrypt");
-    const db = await this.db();
-    const { id, success, message } = await db
-      .run(
-        `INSERT INTO ${this.tablename} (email, password, role) VALUES(?, ?, ?)`,
-        [email, bcrypt.hashSync(password, 10), role ?? "default"]
-      )
-      .then((res) => {
-        return { id: res.lastID, success: true, message: null };
-      })
-      .catch((error) => {
-        return { id: null, success: false, message: error.message };
+    try {
+      const bcrypt = require("bcrypt");
+      const user = await prisma.users.create({
+        data: {
+          email,
+          password: bcrypt.hashSync(password, 10),
+          role: role ?? "default",
+        },
       });
 
-    if (!success) {
-      await db.close();
-      console.error("FAILED TO CREATE USER.", message);
-      return { user: null, message };
+      return { user, message: null };
+    } catch (e) {
+      console.error("FAILED TO CREATE USER.", error.message);
+      return { user: null, error: error.message };
     }
-
-    const user = await db.get(
-      `SELECT * FROM ${this.tablename} WHERE id = ${id} `
-    );
-    await db.close();
-
-    return { user, message: null };
   },
-  get: async function (clause = "") {
-    const db = await this.db();
-    const result = await db
-      .get(`SELECT * FROM ${this.tablename} WHERE ${clause} `)
-      .then((res) => res || null);
-    if (!result) return null;
-    await db.close();
 
-    return result;
-  },
-  where: async function (clause = null, limit = null) {
-    const db = await this.db();
-    const results = await db.all(
-      `SELECT * FROM ${this.tablename} ${clause ? `WHERE ${clause}` : ""} ${
-        !!limit ? `LIMIT ${limit}` : ""
-      } `
-    );
-    await db.close();
-
-    return results;
-  },
-  count: async function (clause = null) {
-    const db = await this.db();
-    const { count } = await db.get(
-      `SELECT COUNT(*) as count FROM ${this.tablename} ${
-        clause ? `WHERE ${clause}` : ""
-      } `
-    );
-    await db.close();
-
-    return count;
-  },
-  whereWithOrgs: async function (clause = null, limit = null) {
-    const { Organization } = require("./organization");
-    const { OrganizationUser } = require("./organizationUser");
-
-    const users = await this.where(clause, limit);
-    const organizations = await Organization.where();
-
-    for (const user of users) {
-      const memberships = await OrganizationUser.where(`user_id = ${user.id}`);
-      delete user.password;
-      user.memberships = [];
-
-      for (const membership of memberships) {
-        const org = organizations.find(
-          (org) => org.id === membership.organization_id
-        );
-        user.memberships.push({ ...org, organization_id: org.id });
-      }
+  get: async function (clause = {}) {
+    try {
+      const user = await prisma.users.findFirst({ where: clause });
+      return user ? { ...user } : null;
+    } catch (error) {
+      console.error(error.message);
+      return null;
     }
-
-    return users;
   },
-  addToAllOrgs: async function (userId = null) {
-    if (!userId) return false;
-    const { Organization } = require("./organization");
-    const { OrganizationUser } = require("./organizationUser");
 
-    const organizations = await Organization.where();
-    if (!organizations.length) return;
+  where: async function (clause = {}, limit = null) {
+    try {
+      const users = await prisma.users.findMany({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+      });
+      return users;
+    } catch (error) {
+      console.error(error.message);
+      return [];
+    }
+  },
 
-    const orgIds = organizations.map((org) => org.id);
-    await OrganizationUser.createMany(userId, orgIds);
-    return;
+  count: async function (clause = {}) {
+    try {
+      const count = await prisma.users.count({ where: clause });
+      return count;
+    } catch (error) {
+      console.error(error.message);
+      return 0;
+    }
   },
-  delete: async function (clause = null) {
-    const db = await this.db();
-    await db.exec(`DELETE FROM ${this.tablename} WHERE ${clause}`);
-    await db.close();
-    return;
+
+  // whereWithOrgs: async function (clause = null, limit = null) {
+  //   const { Organization } = require("./organization");
+  //   const { OrganizationUser } = require("./organizationUser");
+
+  //   const users = await this.where(clause, limit);
+  //   const organizations = await Organization.where();
+
+  //   for (const user of users) {
+  //     const memberships = await OrganizationUser.where(`user_id = ${user.id}`);
+  //     delete user.password;
+  //     user.memberships = [];
+
+  //     for (const membership of memberships) {
+  //       const org = organizations.find(
+  //         (org) => org.id === membership.organization_id
+  //       );
+  //       user.memberships.push({ ...org, organization_id: org.id });
+  //     }
+  //   }
+
+  //   return users;
+  // },
+  // addToAllOrgs: async function (userId = null) {
+  //   if (!userId) return false;
+  //   const { Organization } = require("./organization");
+  //   const { OrganizationUser } = require("./organizationUser");
+
+  //   const organizations = await Organization.where();
+  //   if (!organizations.length) return;
+
+  //   const orgIds = organizations.map((org) => org.id);
+  //   await OrganizationUser.createMany(userId, orgIds);
+  //   return;
+  // },
+
+  delete: async function (clause = {}) {
+    try {
+      await prisma.users.delete({ where: clause });
+      return true;
+    } catch (error) {
+      console.error(error.message);
+      return false;
+    }
   },
+
   update: async function (userId, updates = {}) {
-    const user = await this.get(`id = ${userId}`);
-    if (!user) return { success: false, error: "User does not exist." };
-    const { email, password, role, memberships } = updates;
-    const toUpdate = {};
+    try {
+      const user = await this.get({ id: parseInt(userId) });
+      if (!user) return { success: false, error: "User does not exist." };
 
-    if (user.email !== email && email?.length > 0) {
-      const usedEmail = !!(await this.get(`email = '${email}'`));
-      if (usedEmail)
-        return { success: false, error: `${email} is already in use.` };
-      toUpdate.email = email;
-    }
-
-    if (!!password) {
-      const bcrypt = require("bcrypt");
-      toUpdate.password = bcrypt.hashSync(password, 10);
-    }
-
-    if (user.role !== role && ["admin", "default"].includes(role)) {
-      // If was existing admin and that has been changed
-      // make sure at least one admin exists
-      if (user.role === "admin") {
-        const validAdminCount = (await this.count(`role = 'admin'`)) > 1;
-        if (!validAdminCount)
-          return {
-            success: false,
-            error: `There would be no admins if this action was completed. There must be at least one admin.`,
-          };
+      const { email, password, role, memberships } = updates;
+      const toUpdate = {};
+      if (user.email !== email && email?.length > 0) {
+        const usedEmail = !!(await this.get({ email }));
+        if (usedEmail)
+          return { success: false, error: `${email} is already in use.` };
+        toUpdate.email = email;
       }
 
-      toUpdate.role = role;
-    }
-
-    if (Object.keys(toUpdate).length !== 0) {
-      const values = Object.values(toUpdate);
-      const template = `UPDATE ${this.tablename} SET ${Object.keys(
-        toUpdate
-      ).map((key) => {
-        return `${key}=?`;
-      })} WHERE id = ?`;
-
-      const db = await this.db();
-      const { success, message } = await db
-        .run(template, [...values, userId])
-        .then(() => {
-          return { success: true, message: null };
-        })
-        .catch((error) => {
-          return { success: false, message: error.message };
-        });
-
-      await db.close();
-      if (!success) {
-        console.error(message);
-        return { success: false, error: message };
+      if (!!password) {
+        const bcrypt = require("bcrypt");
+        toUpdate.password = bcrypt.hashSync(password, 10);
       }
-    }
 
-    const { OrganizationUser } = require("./organizationUser");
-    await OrganizationUser.updateOrgPermissions(user.id, memberships);
-    return { success: true, error: null };
+      if (user.role !== role && ["admin", "default"].includes(role)) {
+        // If was existing admin and that has been changed
+        // make sure at least one admin exists
+        if (user.role === "admin") {
+          const validAdminCount = (await this.count({ role: "admin" })) > 1;
+          if (!validAdminCount)
+            return {
+              success: false,
+              error: `There would be no admins if this action was completed. There must be at least one admin.`,
+            };
+        }
+        toUpdate.role = role;
+      }
+
+      const updatedUser = await prisma.users.update({
+        where: { id: parseInt(userId) },
+        data: toUpdate,
+      });
+
+      const { OrganizationUser } = require("./organizationUser");
+      await OrganizationUser.updateOrgPermissions(user.id, memberships);
+      return { success: !!updatedUser, error: null };
+    } catch (error) {
+      console.error(error.message);
+      return { success: false, error: error.message };
+    }
   },
 };
 

backend/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@pinecone-database/pinecone": "^0.1.6",
+    "@prisma/client": "^5.3.1",
     "@qdrant/js-client-rest": "^1.5.0",
     "bcrypt": "^5.1.0",
     "body-parser": "^1.20.2",
@@ -34,6 +35,7 @@
     "openai": "^3.3.0",
     "pinecone-client": "^1.1.0",
     "posthog-node": "^3.1.2",
+    "prisma": "^5.3.1",
     "slugify": "^1.6.6",
     "sqlite": "^4.2.1",
     "sqlite3": "^5.1.6",

backend/prisma/migrations/20230927213910_init/migration.sql

@@ -0,0 +1,183 @@
+-- CreateTable
+CREATE TABLE "users" (
+    "id" SERIAL NOT NULL,
+    "email" TEXT NOT NULL,
+    "password" TEXT NOT NULL,
+    "role" TEXT NOT NULL DEFAULT 'default',
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "users_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "organizations" (
+    "id" SERIAL NOT NULL,
+    "name" TEXT NOT NULL,
+    "slug" TEXT NOT NULL,
+    "uuid" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "organizations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "organization_users" (
+    "id" SERIAL NOT NULL,
+    "user_id" INTEGER NOT NULL,
+    "organization_id" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "organization_users_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "organization_api_keys" (
+    "id" SERIAL NOT NULL,
+    "apiKey" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "organization_id" INTEGER NOT NULL,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "organization_api_keys_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "organization_connections" (
+    "id" SERIAL NOT NULL,
+    "type" TEXT NOT NULL,
+    "settings" TEXT NOT NULL,
+    "organization_id" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "organization_connections_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "organization_workspaces" (
+    "id" SERIAL NOT NULL,
+    "type" TEXT NOT NULL,
+    "settings" TEXT NOT NULL,
+    "uuid" TEXT NOT NULL,
+    "organization_id" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "organization_workspaces_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "workspace_documents" (
+    "id" SERIAL NOT NULL,
+    "name" TEXT NOT NULL,
+    "docId" TEXT NOT NULL,
+    "organization_id" INTEGER NOT NULL,
+    "workspace_id" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "workspace_documents_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "document_vectors" (
+    "id" SERIAL NOT NULL,
+    "docId" TEXT NOT NULL,
+    "vectorId" TEXT NOT NULL,
+    "document_id" INTEGER NOT NULL,
+    "workspace_id" INTEGER NOT NULL,
+    "organization_id" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "document_vectors_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "jobs" (
+    "id" SERIAL NOT NULL,
+    "taskName" TEXT NOT NULL,
+    "status" TEXT NOT NULL DEFAULT 'pending',
+    "data" TEXT NOT NULL,
+    "result" TEXT NOT NULL,
+    "run_by_user_id" INTEGER NOT NULL,
+    "organization_id" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "jobs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "system_settings" (
+    "id" SERIAL NOT NULL,
+    "label" TEXT NOT NULL,
+    "value" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "system_settings_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "users_email_key" ON "users"("email");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "organizations_slug_key" ON "organizations"("slug");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "organizations_uuid_key" ON "organizations"("uuid");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "organization_api_keys_apiKey_key" ON "organization_api_keys"("apiKey");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "organization_workspaces_settings_key" ON "organization_workspaces"("settings");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "organization_workspaces_uuid_key" ON "organization_workspaces"("uuid");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "workspace_documents_docId_key" ON "workspace_documents"("docId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "system_settings_label_key" ON "system_settings"("label");
+
+-- AddForeignKey
+ALTER TABLE "organization_users" ADD CONSTRAINT "organization_users_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "organization_users" ADD CONSTRAINT "organization_users_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "organization_api_keys" ADD CONSTRAINT "organization_api_keys_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "organization_connections" ADD CONSTRAINT "organization_connections_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "organization_workspaces" ADD CONSTRAINT "organization_workspaces_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "workspace_documents" ADD CONSTRAINT "workspace_documents_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "workspace_documents" ADD CONSTRAINT "workspace_documents_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "organization_workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "document_vectors" ADD CONSTRAINT "document_vectors_document_id_fkey" FOREIGN KEY ("document_id") REFERENCES "workspace_documents"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "document_vectors" ADD CONSTRAINT "document_vectors_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "organization_workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "document_vectors" ADD CONSTRAINT "document_vectors_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_run_by_user_id_fkey" FOREIGN KEY ("run_by_user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "jobs" ADD CONSTRAINT "jobs_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;

backend/prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "postgresql"

backend/prisma/schema.prisma

@@ -0,0 +1,129 @@
+generator client {
+  provider = "prisma-client-js"
+}
+
+// Make sure to set the correct DATABASE_URL in your .env file
+// After swapping run `yarn prisma:setup` from the root directory to migrate the database
+datasource db {
+  provider = "postgresql"
+  url      = "postgresql://tim:tdc1994@localhost:5432/vdbms"
+}
+
+model users {
+  id                 Int                  @id @default(autoincrement())
+  email              String               @unique
+  password           String
+  role               String               @default("default")
+  createdAt          DateTime             @default(now())
+  lastUpdatedAt      DateTime             @default(now())
+  organization_users organization_users[]
+  jobs               jobs[]
+}
+
+model organizations {
+  id                       Int                        @id @default(autoincrement())
+  name                     String
+  slug                     String                     @unique
+  uuid                     String                     @unique
+  createdAt                DateTime                   @default(now())
+  lastUpdatedAt            DateTime                   @default(now())
+  organization_users       organization_users[]
+  organization_api_keys    organization_api_keys[]
+  organization_connections organization_connections[]
+  organization_workspaces  organization_workspaces[]
+  workspace_documents      workspace_documents[]
+  document_vectors         document_vectors[]
+  jobs                     jobs[]
+}
+
+model organization_users {
+  id              Int           @id @default(autoincrement())
+  user_id         Int
+  organization_id Int
+  createdAt       DateTime      @default(now())
+  lastUpdatedAt   DateTime      @default(now())
+  user            users         @relation(fields: [user_id], references: [id])
+  organization    organizations @relation(fields: [organization_id], references: [id])
+}
+
+model organization_api_keys {
+  id              Int           @id @default(autoincrement())
+  apiKey          String        @unique
+  createdAt       DateTime      @default(now())
+  organization_id Int
+  lastUpdatedAt   DateTime      @default(now())
+  organization    organizations @relation(fields: [organization_id], references: [id], onDelete: Cascade)
+}
+
+model organization_connections {
+  id              Int           @id @default(autoincrement())
+  type            String
+  settings        String
+  organization_id Int
+  createdAt       DateTime      @default(now())
+  lastUpdatedAt   DateTime      @default(now())
+  organization    organizations @relation(fields: [organization_id], references: [id], onDelete: Cascade)
+}
+
+model organization_workspaces {
+  id                  Int                   @id @default(autoincrement())
+  type                String
+  settings            String                @unique
+  uuid                String                @unique
+  organization_id     Int
+  createdAt           DateTime              @default(now())
+  lastUpdatedAt       DateTime              @default(now())
+  organization        organizations         @relation(fields: [organization_id], references: [id], onDelete: Cascade)
+  workspace_documents workspace_documents[]
+  document_vectors    document_vectors[]
+}
+
+model workspace_documents {
+  id               Int                     @id @default(autoincrement())
+  name             String
+  docId            String                  @unique
+  organization_id  Int
+  workspace_id     Int
+  createdAt        DateTime                @default(now())
+  lastUpdatedAt    DateTime                @default(now())
+  organization     organizations           @relation(fields: [organization_id], references: [id], onDelete: Cascade)
+  workspace        organization_workspaces @relation(fields: [workspace_id], references: [id], onDelete: Cascade)
+  document_vectors document_vectors[]
+}
+
+model document_vectors {
+  id              Int      @id @default(autoincrement())
+  docId           String
+  vectorId        String
+  document_id     Int
+  workspace_id    Int
+  organization_id Int
+  createdAt       DateTime @default(now())
+  lastUpdatedAt   DateTime @default(now())
+
+  document     workspace_documents     @relation(fields: [document_id], references: [id], onDelete: Cascade)
+  workspace    organization_workspaces @relation(fields: [workspace_id], references: [id], onDelete: Cascade)
+  organization organizations           @relation(fields: [organization_id], references: [id], onDelete: Cascade)
+}
+
+model jobs {
+  id              Int           @id @default(autoincrement())
+  taskName        String
+  status          String        @default("pending")
+  data            String
+  result          String
+  run_by_user_id  Int
+  organization_id Int
+  createdAt       DateTime      @default(now())
+  lastUpdatedAt   DateTime      @default(now())
+  user            users         @relation(fields: [run_by_user_id], references: [id], onDelete: Cascade)
+  organization    organizations @relation(fields: [organization_id], references: [id], onDelete: Cascade)
+}
+
+model system_settings {
+  id            Int      @id @default(autoincrement())
+  label         String   @unique
+  value         String
+  createdAt     DateTime @default(now())
+  lastUpdatedAt DateTime @default(now())
+}

backend/utils/boot/index.js

@@ -16,29 +16,29 @@ const { User } = require("../../models/user");
 const { WorkspaceDocument } = require("../../models/workspaceDocument");
 const { getGitVersion } = require("../metrics");
 
-function findOrCreateDBFile() {
-  const fs = require("fs");
-  const path = require("path");
-  const storageFolder = path.resolve(__dirname, `../../storage/`);
-  const dbPath = `${storageFolder}/vdbms.db`;
-  if (!fs.existsSync(storageFolder)) fs.mkdirSync(storageFolder);
-  if (fs.existsSync(dbPath)) return;
-  fs.writeFileSync(dbPath, "");
-  console.log("SQLite db created on boot.");
-  return;
-}
+// function findOrCreateDBFile() {
+//   const fs = require("fs");
+//   const path = require("path");
+//   const storageFolder = path.resolve(__dirname, `../../storage/`);
+//   const dbPath = `${storageFolder}/vdbms.db`;
+//   if (!fs.existsSync(storageFolder)) fs.mkdirSync(storageFolder);
+//   if (fs.existsSync(dbPath)) return;
+//   fs.writeFileSync(dbPath, "");
+//   console.log("SQLite db created on boot.");
+//   return;
+// }
 
-function findOrCreateJobDBFile() {
-  const path = require("path");
-  const fs = require("fs");
-  const storageFolder = path.resolve(__dirname, `../../storage/`);
-  const dbPath = `${storageFolder}/job_queue.db`;
-  if (!fs.existsSync(storageFolder)) fs.mkdirSync(storageFolder);
-  if (fs.existsSync(dbPath)) return;
-  fs.writeFileSync(dbPath, "");
-  console.log("SQLite jobs db created on boot.");
-  return;
-}
+// function findOrCreateJobDBFile() {
+//   const path = require("path");
+//   const fs = require("fs");
+//   const storageFolder = path.resolve(__dirname, `../../storage/`);
+//   const dbPath = `${storageFolder}/job_queue.db`;
+//   if (!fs.existsSync(storageFolder)) fs.mkdirSync(storageFolder);
+//   if (fs.existsSync(dbPath)) return;
+//   fs.writeFileSync(dbPath, "");
+//   console.log("SQLite jobs db created on boot.");
+//   return;
+// }
 
 function setupVectorCacheStorage() {
   const fs = require("fs");
@@ -52,18 +52,18 @@ function setupVectorCacheStorage() {
 
 // Init all tables so to not try to reference foreign key
 // tables that may not exist and also have their schema available.
-async function initTables() {
-  (await SystemSettings.db()).close();
-  (await User.db()).close();
-  (await Organization.db()).close();
-  (await OrganizationApiKey.db()).close();
-  (await OrganizationConnection.db()).close();
-  (await OrganizationUser.db()).close();
-  (await OrganizationWorkspace.db()).close();
-  (await WorkspaceDocument.db()).close();
-  (await DocumentVectors.db()).close();
-  (await Queue.db()).close();
-}
+// async function initTables() {
+//   (await SystemSettings.db()).close();
+//   (await User.db()).close();
+//   (await Organization.db()).close();
+//   (await OrganizationApiKey.db()).close();
+//   (await OrganizationConnection.db()).close();
+//   (await OrganizationUser.db()).close();
+//   (await OrganizationWorkspace.db()).close();
+//   (await WorkspaceDocument.db()).close();
+//   (await DocumentVectors.db()).close();
+//   (await Queue.db()).close();
+// }
 
 // Telemetry is anonymized and your data is never read. This can be disabled by setting
 // DISABLE_TELEMETRY=true in the `.env` of however you setup. Telemetry helps us determine use
@@ -96,39 +96,30 @@ async function setupTelemetry() {
 
 async function systemInit() {
   try {
-    await findOrCreateDBFile();
-    await findOrCreateJobDBFile();
-    await setupVectorCacheStorage();
-    await initTables();
+    setupVectorCacheStorage();
     await setupTelemetry();
-    const completeSetup = (await User.count('role = "admin"')) > 0;
+    const completeSetup = (await User.count({ role: "admin" })) > 0;
     if (completeSetup) return;
 
     process.env.SYS_EMAIL = process.env.SYS_EMAIL ?? "root@vectoradmin.com";
     process.env.SYS_PASSWORD = process.env.SYS_PASSWORD ?? "password";
 
-    const existingRootUser = await User.get(
-      `email = '${process.env.SYS_EMAIL}' AND role = 'root'`
-    );
+    const existingRootUser = await User.get({
+      email: process.env.SYS_EMAIL,
+      role: "root",
+    });
     if (existingRootUser) return;
 
     const bcrypt = require("bcrypt");
-    const userDb = await User.db();
-    const { success, message } = await userDb
-      .run(
-        `INSERT INTO ${User.tablename} (email, password, role) VALUES (?, ?, 'root')`,
-        [process.env.SYS_EMAIL, bcrypt.hashSync(process.env.SYS_PASSWORD, 10)]
-      )
-      .then((res) => {
-        return { id: res.lastID, success: true, message: null };
-      })
-      .catch((error) => {
-        return { id: null, success: false, message: error.message };
-      });
+    const rootUser = await User.create({
+      email: process.env.SYS_EMAIL,
+      password: bcrypt.hashSync(process.env.SYS_PASSWORD, 10),
+      role: "root",
+    });
 
-    if (!success) {
-      await userDb.close();
-      console.error("FAILED TO CREATE ROOT USER.", message);
+    if (!rootUser) {
+      console.error("FAILED TO CREATE ROOT USER!", message);
+      return;
     }
 
     console.log("Root user created with credentials");

backend/utils/http/index.js

@@ -61,7 +61,7 @@ async function userFromSession(request) {
     return null;
   }
 
-  const user = await User.get(`id = ${valid.id}`);
+  const user = await User.get({ id: Number(valid.id) });
   return user;
 }
 

backend/utils/prisma/PRISMA.md

@@ -0,0 +1,47 @@
+# Prisma Setup and Usage Guide
+
+This guide will help you set up and use Prisma for the project. Prisma is a powerful ORM for Node.js and TypeScript, helping developers build faster and make fewer errors. Follow the guide to understand how to use Prisma and the scripts available in the project to manage the Prisma setup.
+
+## Setting Up Prisma
+
+To get started with setting up Prisma, you should run the setup script from the project root directory:
+
+```sh
+yarn setup
+```
+
+This script will install the necessary node modules in both the server and frontend directories, set up the environment files, and set up Prisma (generate client, run migrations, and seed the database).
+
+## Prisma Scripts
+
+In the project root's `package.json`, there are several scripts set up to help you manage Prisma:
+
+- **prisma:generate**: Generates the Prisma client.
+- **prisma:migrate**: Runs the migrations to ensure the database is in sync with the schema.
+- **prisma:seed**: Seeds the database with initial data.
+- **prisma:setup**: A convenience script that runs `prisma:generate`, `prisma:migrate`, and `prisma:seed` in sequence.
+- **sqlite:migrate**: (To be run from the `server` directory) This script is for users transitioning from the old SQLite custom ORM setup to Prisma and will migrate all exisiting data over to Prisma. If you're a new user, your setup will already use Prisma.
+
+To run any of these scripts, use `yarn` followed by the script name from the project root directory. For example:
+
+```sh
+yarn prisma:setup
+```
+
+## Manual Prisma Commands
+
+While the scripts should cover most of your needs, you may sometimes want to run Prisma commands manually. Here are some commands you might find useful, along with their descriptions:
+
+- `npx prisma introspect`: Introspects the database to update the Prisma schema by reading the schema of the existing database.
+- `npx prisma generate`: Generates the Prisma client.
+- `npx prisma migrate dev --name init`: Ensures the database is in sync with the schema, naming the migration 'init'.
+- `npx prisma migrate reset`: Resets the database, deleting all data and recreating the schema.
+
+These commands should be run from the `server` directory, where the Prisma schema is located.
+
+## Notes
+
+- Always make sure to run scripts from the root level to avoid path issues.
+- Before running migrations, ensure that the Prisma schema is correctly defined to prevent data loss or corruption.
+- If you are adding a new feature or making changes that require a change in the database schema, create a new migration rather than editing existing migrations.
+- For users transitioning from the old SQLite ORM, navigate to the `server` directory and run the `sqlite:migrate` script to smoothly transition to Prisma. If you're setting up the project fresh, this step is unnecessary as the setup will already be using Prisma.

backend/utils/prisma/index.js

@@ -0,0 +1,12 @@
+const { PrismaClient } = require("@prisma/client");
+
+// npx prisma introspect
+// npx prisma generate
+// npx prisma migrate dev --name init -> ensures that db is in sync with schema
+// npx prisma migrate reset -> resets the db
+
+const prisma = new PrismaClient({
+  log: ["query", "info", "warn"],
+});
+
+module.exports = prisma;

backend/utils/search/documentEmbeddings/semantic.js

@@ -19,7 +19,7 @@ async function semanticSearch(document, query) {
   if (!connector)
     return { fragments: [], error: "No connector found for org." };
 
-  const openAiKey = (await SystemSettings.get(`label = 'open_ai_api_key'`))
+  const openAiKey = (await SystemSettings.get({ label: "open_ai_api_key" }))
     ?.value;
   if (!openAiKey)
     return { fragments: [], error: "No OpenAI key available to embed query." };

backend/utils/search/workspaceDocuments/semantic.js

@@ -14,7 +14,7 @@ async function semanticSearch(workspace, query) {
   if (!connector)
     return { documents: [], error: "No connector found for org." };
 
-  const openAiKey = (await SystemSettings.get(`label = 'open_ai_api_key'`))
+  const openAiKey = (await SystemSettings.get({ label: "open_ai_api_key" }))
     ?.value;
   if (!openAiKey)
     return { documents: [], error: "No OpenAI key available to embed query." };

package.json

@@ -12,7 +12,10 @@
     "dev:server": "cd backend && yarn dev",
     "dev:frontend": "cd frontend && yarn start",
     "dev:workers": "cd workers && ./dev_entrypoint.sh",
-    "lint": "cd backend && yarn lint && cd ../frontend && yarn lint && cd ../workers && yarn lint"
+    "lint": "cd backend && yarn lint && cd ../frontend && yarn lint && cd ../workers && yarn lint",
+    "prisma:generate": "cd backend && npx prisma generate",
+    "prisma:migrate": "cd backend && npx prisma migrate dev --name init",
+    "prisma:setup": "yarn prisma:generate && yarn prisma:migrate"
   },
   "private": false
 }