simplify tiktoken import for lighter import

frontend/.gitignore

@@ -23,4 +23,5 @@ dist-ssr
 *.sln
 *.sw?
 legacysrc
-.env.production
+.env.production
+bundle.html

frontend/package.json

@@ -6,7 +6,8 @@
     "start": "vite --open",
     "build": "vite build",
     "preview": "vite preview",
-    "lint": "yarn prettier --write ./src"
+    "lint": "yarn prettier --write ./src",
+    "dev:build": "npx vite-bundle-visualizer -o ./bundle.html --open false && echo \"yarn serve && Open localhost:3000/bundle in browser to view bundle.\""
   },
   "overrides": {
     "react-syntax-highlighter": {
@@ -14,18 +15,14 @@
     }
   },
   "dependencies": {
+    "@dqbd/tiktoken": "^1.0.7",
     "@metamask/jazzicon": "^2.0.0",
-    "apexcharts": "^3.37.0",
-    "js-tiktoken": "^1.0.7",
     "jsvectormap": "^1.5.1",
-    "localforage": "^1.10.0",
     "lodash": "^4.17.21",
     "lodash.debounce": "^4.0.8",
-    "match-sorter": "^6.3.1",
     "moment": "^2.29.4",
     "pluralize": "^8.0.0",
     "react": "^18.2.0",
-    "react-apexcharts": "^1.4.0",
     "react-code-blocks": "^0.0.9-0",
     "react-dom": "^18.2.0",
     "react-dropzone": "^14.2.3",
@@ -34,7 +31,6 @@
     "react-loading-icons": "^1.1.0",
     "react-router-dom": "^6.8.1",
     "react-toastify": "^9.1.3",
-    "sort-by": "^0.0.2",
     "title-case": "^3.0.3",
     "truncate": "^3.0.0",
     "uuid": "^9.0.0"
@@ -49,8 +45,12 @@
     "postcss": "^8.4.21",
     "prettier": "^2.8.4",
     "prettier-plugin-tailwindcss": "^0.2.2",
+    "serve": "^14.2.1",
     "tailwindcss": "^3.2.6",
     "vite": "^4.1.0",
-    "webpack": "^5.79.0"
+    "vite-bundle-visualizer": "^0.10.0",
+    "webpack": "^5.79.0",
+    "vite-plugin-top-level-await": "^1.3.1",
+    "vite-plugin-wasm": "^3.2.2"
   }
 }

frontend/src/pages/DocumentView/FragmentList/EditEmbeddingConfirmation/index.tsx

@@ -150,9 +150,8 @@ const EditEmbeddingConfirmation = memo(
           <div className="flex w-full items-center justify-between px-4">
             <p className="font-semibold text-red-600">{error || ''}</p>
             <p
-              className={`text-sm ${
-                error ? 'font-semibold text-red-600' : 'text-slate-600'
-              }`}
+              className={`text-sm ${error ? 'font-semibold text-red-600' : 'text-slate-600'
+                }`}
             >
               {numberWithCommas(tokenLength)}/
               {numberWithCommas(MAX_TOKENS.cl100k_base)}{' '}

frontend/src/utils/tokenizer/index.ts

@@ -1,4 +1,5 @@
-import { getEncoding } from 'js-tiktoken';
+import { Tiktoken } from "@dqbd/tiktoken/lite";
+import cl100k_base from "@dqbd/tiktoken/encoders/cl100k_base.json";
 
 type IValidEmbeddingModels = 'cl100k_base';
 export const MAX_TOKENS = {
@@ -6,7 +7,11 @@ export const MAX_TOKENS = {
 };
 
 export function countLLMTokens(input: string) {
-  const encoding = getEncoding('cl100k_base');
+  const encoding = new Tiktoken(
+    cl100k_base.bpe_ranks,
+    cl100k_base.special_tokens,
+    cl100k_base.pat_str
+  );
   const tokens = encoding.encode(input);
   return { tokens, length: tokens.length };
 }

frontend/vite.config.js

@@ -1,5 +1,7 @@
 import { defineConfig } from 'vite'
 import react from '@vitejs/plugin-react'
+import wasm from "vite-plugin-wasm";
+import topLevelAwait from "vite-plugin-top-level-await";
 
 // https://vitejs.dev/config/
 export default defineConfig({
@@ -11,5 +13,5 @@ export default defineConfig({
     'process.env': process.env,
     global: 'window'
   },
-  plugins: [react()],
+  plugins: [react(), wasm(), topLevelAwait()],
 })