chore: move multi-model into single sub module (#1443)

.changeset/cool-cows-lick.md

@@ -0,0 +1,8 @@
+---
+"@llamaindex/env": patch
+"llamaindex": patch
+"@llamaindex/clip": patch
+"@llamaindex/huggingface": patch
+---
+
+chore: move multi-model into single sub module

packages/env/multi-model/package.json

@@ -0,0 +1,12 @@
+{
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    "browser": "./dist/index.browser.js",
+    "edge-light": "./dist/index.edge-light.js",
+    "workerd": "./dist/index.workerd.js"
+  },
+  "private": true
+}

packages/env/package.json

@@ -28,7 +28,7 @@
         "types": "./dist/index.d.ts",
         "import": "./dist/index.js",
         "require": "./dist/index.cjs",
-        "default": "./dist/index.cjs"
+        "default": "./dist/index.js"
       },
       "workerd": {
         "types": "./dist/index.workerd.d.ts",
@@ -50,9 +50,36 @@
         "types": "./dist/index.d.cts",
         "default": "./dist/index.cjs"
       }
+    },
+    "./multi-model": {
+      "workerd": {
+        "types": "./multi-model/dist/multi-model.workerd.d.ts",
+        "default": "./multi-model/dist/multi-model.workerd.js"
+      },
+      "edge-light": {
+        "types": "./multi-model/dist/multi-model.edge-light.d.ts",
+        "default": "./multi-model/dist/multi-model.edge-light.js"
+      },
+      "browser": {
+        "types": "./multi-model/dist/multi-model.browser.d.ts",
+        "default": "./multi-model/dist/multi-model.browser.js"
+      },
+      "import": {
+        "types": "./multi-model/dist/multi-model.d.ts",
+        "default": "./multi-model/dist/multi-model.js"
+      },
+      "require": {
+        "types": "./multi-model/dist/multi-model.d.cts",
+        "default": "./multi-model/dist/multi-model.cjs"
+      },
+      "default": {
+        "types": "./multi-model/dist/multi-model.d.ts",
+        "default": "./multi-model/dist/multi-model.js"
+      }
     }
   },
   "files": [
+    "multi-model",
     "dist",
     "CHANGELOG.md",
     "!**/*.tsbuildinfo"

packages/env/src/index.browser.ts

@@ -6,12 +6,6 @@
 import "./global-check.js";
 
 export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
-export {
-  loadTransformers,
-  setTransformers,
-  type LoadTransformerEvent,
-  type OnLoad,
-} from "./multi-model/index.browser.js";
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";
 export { NotSupportCurrentRuntimeClass } from "./utils/shared.js";
 export * from "./web-polyfill.js";

packages/env/src/index.edge-light.ts

@@ -5,12 +5,6 @@
  */
 import "./global-check.js";
 export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
-export {
-  loadTransformers,
-  setTransformers,
-  type LoadTransformerEvent,
-  type OnLoad,
-} from "./multi-model/index.non-nodejs.js";
 export * from "./node-polyfill.js";
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";
 export { NotSupportCurrentRuntimeClass } from "./utils/shared.js";

packages/env/src/index.ts

@@ -35,12 +35,6 @@ export function createSHA256(): SHA256 {
 }
 
 export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
-export {
-  loadTransformers,
-  setTransformers,
-  type LoadTransformerEvent,
-  type OnLoad,
-} from "./multi-model/index.js";
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
 export {
   AsyncLocalStorage,

packages/env/src/index.workerd.ts

@@ -16,10 +16,4 @@ export function getEnv(name: string): string | undefined {
 }
 
 export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
-export {
-  loadTransformers,
-  setTransformers,
-  type LoadTransformerEvent,
-  type OnLoad,
-} from "./multi-model/index.non-nodejs.js";
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";

packages/env/src/multi-model.browser.ts

@@ -0,0 +1,6 @@
+export {
+  loadTransformers,
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./internal/multi-model/browser.js";

packages/env/src/multi-model.edge-light.ts

@@ -0,0 +1,6 @@
+export {
+  loadTransformers,
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./internal/multi-model/non-node.js";

packages/env/src/multi-model.ts

@@ -0,0 +1,6 @@
+export {
+  loadTransformers,
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./internal/multi-model/node.js";

packages/env/src/multi-model.workerd.ts

@@ -0,0 +1,6 @@
+export {
+  loadTransformers,
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./internal/multi-model/non-node.js";

packages/llamaindex/e2e/node/embedding/clip.e2e.ts

@@ -1,5 +1,5 @@
-import type { LoadTransformerEvent } from "@llamaindex/env";
-import { setTransformers } from "@llamaindex/env";
+import type { LoadTransformerEvent } from "@llamaindex/env/multi-model";
+import { setTransformers } from "@llamaindex/env/multi-model";
 import { ClipEmbedding, ImageNode, Settings } from "llamaindex";
 import assert from "node:assert";
 import { type Mock, test } from "node:test";

packages/providers/clip/src/embedding.ts

@@ -2,7 +2,10 @@ import { MultiModalEmbedding } from "@llamaindex/core/embeddings";
 import type { ImageType } from "@llamaindex/core/schema";
 // only import type, to avoid bundling error
 import { Settings } from "@llamaindex/core/global";
-import { type LoadTransformerEvent, loadTransformers } from "@llamaindex/env";
+import {
+  type LoadTransformerEvent,
+  loadTransformers,
+} from "@llamaindex/env/multi-model";
 import type {
   CLIPTextModelWithProjection,
   CLIPVisionModelWithProjection,

packages/providers/huggingface/src/embedding.ts

@@ -1,6 +1,9 @@
 import { BaseEmbedding } from "@llamaindex/core/embeddings";
 import { Settings } from "@llamaindex/core/global";
-import { type LoadTransformerEvent, loadTransformers } from "@llamaindex/env";
+import {
+  type LoadTransformerEvent,
+  loadTransformers,
+} from "@llamaindex/env/multi-model";
 import type { pipeline } from "@xenova/transformers";
 import { HuggingFaceEmbeddingModelType } from "./shared";
 

packages/providers/huggingface/src/llm.ts

@@ -9,7 +9,7 @@ import {
   type LLMChatParamsStreaming,
   type LLMMetadata,
 } from "@llamaindex/core/llms";
-import { loadTransformers } from "@llamaindex/env";
+import { loadTransformers } from "@llamaindex/env/multi-model";
 import type {
   PreTrainedModel,
   PreTrainedTokenizer,