chore: remove gpt-tokenizer (#1815)

.changeset/friendly-kids-push.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Remove gpt-tokenizer from llamaindex

packages/llamaindex/package.json

@@ -29,7 +29,6 @@
     "@types/lodash": "^4.17.7",
     "@types/node": "^22.9.0",
     "ajv": "^8.17.1",
-    "gpt-tokenizer": "^2.6.2",
     "lodash": "^4.17.21",
     "magic-bytes.js": "^1.10.0"
   },

pnpm-lock.yaml

@@ -1036,9 +1036,6 @@ importers:
       ajv:
         specifier: ^8.17.1
         version: 8.17.1
-      gpt-tokenizer:
-        specifier: ^2.6.2
-        version: 2.8.1
       lodash:
         specifier: ^4.17.21
         version: 4.17.21