Merge pull request #62 from Mintplex-Labs/change-frontend-tiktoken-import

Change frontend tiktoken import

.gitignore

@@ -6,4 +6,4 @@ node_modules
 __pycache__
 v-env
 .DS_Store
-yarn.lock
+yarn-error.log

frontend/.gitignore

@@ -23,4 +23,5 @@ dist-ssr
 *.sln
 *.sw?
 legacysrc
-.env.production
+.env.production
+bundle.html

frontend/package.json

@@ -1,12 +1,13 @@
 {
-  "name": "tailadmin-react-free",
+  "name": "vector-admin-frontend",
   "private": true,
   "type": "module",
   "scripts": {
     "start": "vite --open",
     "build": "vite build",
     "preview": "vite preview",
-    "lint": "yarn prettier --write ./src"
+    "lint": "yarn prettier --write ./src",
+    "dev:build": "npx vite-bundle-visualizer -o ./bundle.html --open false && echo \"yarn serve && Open localhost:3000/bundle in browser to view bundle.\""
   },
   "overrides": {
     "react-syntax-highlighter": {
@@ -14,18 +15,14 @@
     }
   },
   "dependencies": {
+    "@dqbd/tiktoken": "^1.0.7",
     "@metamask/jazzicon": "^2.0.0",
-    "apexcharts": "^3.37.0",
-    "js-tiktoken": "^1.0.7",
     "jsvectormap": "^1.5.1",
-    "localforage": "^1.10.0",
     "lodash": "^4.17.21",
     "lodash.debounce": "^4.0.8",
-    "match-sorter": "^6.3.1",
     "moment": "^2.29.4",
     "pluralize": "^8.0.0",
     "react": "^18.2.0",
-    "react-apexcharts": "^1.4.0",
     "react-code-blocks": "^0.0.9-0",
     "react-dom": "^18.2.0",
     "react-dropzone": "^14.2.3",
@@ -34,7 +31,6 @@
     "react-loading-icons": "^1.1.0",
     "react-router-dom": "^6.8.1",
     "react-toastify": "^9.1.3",
-    "sort-by": "^0.0.2",
     "title-case": "^3.0.3",
     "truncate": "^3.0.0",
     "uuid": "^9.0.0"
@@ -49,8 +45,12 @@
     "postcss": "^8.4.21",
     "prettier": "^2.8.4",
     "prettier-plugin-tailwindcss": "^0.2.2",
+    "serve": "^14.2.1",
     "tailwindcss": "^3.2.6",
     "vite": "^4.1.0",
-    "webpack": "^5.79.0"
+    "vite-bundle-visualizer": "^0.10.0",
+    "webpack": "^5.79.0",
+    "vite-plugin-top-level-await": "^1.3.1",
+    "vite-plugin-wasm": "^3.2.2"
   }
 }

frontend/src/utils/tokenizer/index.ts

@@ -1,4 +1,5 @@
-import { getEncoding } from 'js-tiktoken';
+import { Tiktoken } from '@dqbd/tiktoken/lite';
+import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json';
 
 type IValidEmbeddingModels = 'cl100k_base';
 export const MAX_TOKENS = {
@@ -6,7 +7,11 @@ export const MAX_TOKENS = {
 };
 
 export function countLLMTokens(input: string) {
-  const encoding = getEncoding('cl100k_base');
+  const encoding = new Tiktoken(
+    cl100k_base.bpe_ranks,
+    cl100k_base.special_tokens,
+    cl100k_base.pat_str
+  );
   const tokens = encoding.encode(input);
   return { tokens, length: tokens.length };
 }

frontend/vite.config.js

@@ -1,5 +1,7 @@
 import { defineConfig } from 'vite'
 import react from '@vitejs/plugin-react'
+import wasm from "vite-plugin-wasm";
+import topLevelAwait from "vite-plugin-top-level-await";
 
 // https://vitejs.dev/config/
 export default defineConfig({
@@ -11,5 +13,5 @@ export default defineConfig({
     'process.env': process.env,
     global: 'window'
   },
-  plugins: [react()],
+  plugins: [react(), wasm(), topLevelAwait()],
 })

workers/yarn-error.log

@@ -1,48 +0,0 @@
-Arguments: 
-  /Users/tim/.nvm/versions/node/v18.13.0/bin/node /usr/local/Cellar/yarn/1.22.19/libexec/bin/yarn.js
-
-PATH: 
-  /Users/tim/.rvm/gems/ruby-2.7.3/bin:/Users/tim/.rvm/gems/ruby-2.7.3@global/bin:/Users/tim/.rvm/rubies/ruby-2.7.3/bin:/Library/Frameworks/Python.framework/Versions/3.7/bin:/Users/tim/.local/share/solana/install/active_release/bin:/Users/tim/google-cloud-sdk/bin:/Users/tim/Library/Android/sdk/platform-tools:/Users/tim/.nvm/versions/node/v18.13.0/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:/Users/tim/.rvm/bin:/Users/tim/.foundry/bin
-
-Yarn version: 
-  1.22.19
-
-Node version: 
-  18.13.0
-
-Platform: 
-  darwin x64
-
-Trace: 
-  SyntaxError: /Users/tim/Documents/vdms/workers/package.json: Unexpected token } in JSON at position 385
-      at JSON.parse (<anonymous>)
-      at /usr/local/Cellar/yarn/1.22.19/libexec/lib/cli.js:1629:59
-      at Generator.next (<anonymous>)
-      at step (/usr/local/Cellar/yarn/1.22.19/libexec/lib/cli.js:310:30)
-      at /usr/local/Cellar/yarn/1.22.19/libexec/lib/cli.js:321:13
-
-npm manifest: 
-  {
-    "name": "vector-data-management-system-background-workers",
-    "version": "0.0.1-beta",
-    "description": "",
-    "author": "Timothy Carambat (Mintplex Labs)",
-    "license": "MIT",
-    "engines": {
-      "node": ">=18"
-    },
-    "scripts": {
-      "start": "node devSetup.js",
-      "lint": "yarn prettier --write ./src"
-    },
-    "private": false,
-    "devDependencies": {
-      "prettier": "^2.8.4",
-    }
-  }
-
-yarn manifest: 
-  No manifest
-
-Lockfile: 
-  No lockfile