Fix liger orpo trainer import error (#459)

## Summary
<!--- This is a required section; please describe the main purpose of
this proposed code change. --->

<!---
## Details
This is an optional section; is there anything specific that reviewers
should be aware of?
--->

To fix https://github.com/linkedin/Liger-Kernel/issues/458

1. Move orpo trainer to trainer/
2. Remove trl from `[dev]` now so the test represents the actual user
environment.
3. Disable Qwen VL test for transformers>=4.47.0 for now and create
issue https://github.com/linkedin/Liger-Kernel/issues/461
## Testing Done
<!--- This is a required section; please describe how this change was
tested. --->

<!-- 
Replace BLANK with your device type. For example, A100-80G-PCIe

Complete the following tasks before sending your PR, and replace `[ ]`
with
`[x]` to indicate you have done them. 
-->

- Hardware Type: <BLANK>
- [ ] run `make test` to ensure correctness
- [ ] run `make checkstyle` to ensure code style
- [ ] run `make test-convergence` to ensure convergence

dev/modal/tests.py

@@ -4,6 +4,7 @@ from pathlib import Path
 import modal
 
 ROOT_PATH = Path(__file__).parent.parent.parent
+REMOTE_ROOT_PATH = "/root/liger-kernel"
 
 # REBUILD_IMAGE is an environment variable that is set to "true" in the nightly build
 REBUILD_IMAGE = os.getenv("REBUILD_IMAGE") is not None
@@ -17,13 +18,13 @@ image = modal.Image.debian_slim().pip_install_from_pyproject(
 app = modal.App("liger_tests", image=image)
 
 # mount: add local files to the remote container
-repo = modal.Mount.from_local_dir(ROOT_PATH, remote_path="/root/liger-kernel")
+repo = modal.Mount.from_local_dir(ROOT_PATH, remote_path=REMOTE_ROOT_PATH)
 
 
 @app.function(gpu="A10G", mounts=[repo], timeout=60 * 15)
 def liger_tests():
     import subprocess
 
-    subprocess.run(["pip", "install", "-e", "."], check=True, cwd="/root/liger-kernel")
-    subprocess.run(["make", "test"], check=True, cwd="/root/liger-kernel")
-    subprocess.run(["make", "test-convergence"], check=True, cwd="/root/liger-kernel")
+    subprocess.run(["pip", "install", "-e", "."], check=True, cwd=REMOTE_ROOT_PATH)
+    subprocess.run(["make", "test"], check=True, cwd=REMOTE_ROOT_PATH)
+    subprocess.run(["make", "test-convergence"], check=True, cwd=REMOTE_ROOT_PATH)

dev/modal/tests_bwd.py

@@ -4,6 +4,7 @@ from pathlib import Path
 import modal
 
 ROOT_PATH = Path(__file__).parent.parent.parent
+REMOTE_ROOT_PATH = "/root/liger-kernel"
 
 # REBUILD_IMAGE is an environment variable that is set to "true" in the nightly build
 REBUILD_IMAGE = os.getenv("REBUILD_IMAGE") is not None
@@ -22,13 +23,13 @@ image = (
 app = modal.App("liger_tests", image=image)
 
 # mount: add local files to the remote container
-repo = modal.Mount.from_local_dir(ROOT_PATH, remote_path="/root/liger-kernel")
+repo = modal.Mount.from_local_dir(ROOT_PATH, remote_path=REMOTE_ROOT_PATH)
 
 
 @app.function(gpu="A10G", mounts=[repo], timeout=60 * 10)
 def liger_tests_bwd():
     import subprocess
 
-    subprocess.run(["pip", "install", "-e", "."], check=True, cwd="/root/liger-kernel")
-    subprocess.run(["make", "test"], check=True, cwd="/root/liger-kernel")
-    subprocess.run(["make", "test-convergence"], check=True, cwd="/root/liger-kernel")
+    subprocess.run(["pip", "install", "-e", "."], check=True, cwd=REMOTE_ROOT_PATH)
+    subprocess.run(["make", "test"], check=True, cwd=REMOTE_ROOT_PATH)
+    subprocess.run(["make", "test-convergence"], check=True, cwd=REMOTE_ROOT_PATH)

examples/alignment/run_orpo.py

@@ -1,9 +1,9 @@
 import torch
 from datasets import load_dataset
 from transformers import AutoModelForCausalLM, AutoTokenizer
-from trl import ORPOConfig, ORPOTrainer  # noqa: F401
+from trl import ORPOConfig  # noqa: F401
 
-from liger_kernel.transformers import LigerORPOTrainer  # noqa: F401
+from liger_kernel.transformers.trainer import LigerORPOTrainer  # noqa: F401
 
 model = AutoModelForCausalLM.from_pretrained(
     "meta-llama/Llama-3.2-1B-Instruct",
@@ -19,13 +19,6 @@ tokenizer.pad_token = tokenizer.eos_token
 
 train_dataset = load_dataset("trl-lib/tldr-preference", split="train")
 
-# train_dataset = train_dataset.map(
-#     lambda example: {
-#         "prompt": example["prompt"],
-#         "chosen": example["chosen"][0]["content"],
-#         "rejected": example["rejected"][0]["content"],
-#     }
-# )
 training_args = ORPOConfig(
     output_dir="Llama3.2_1B_Instruct",
     beta=0.1,

pyproject.toml

@@ -20,9 +20,12 @@ transformers = [
     "transformers~=4.0"
 ]
 
+trl = [
+    "trl>=0.11.0",
+]
+
 dev = [
     "transformers>=4.44.2",
-    "trl>=0.11.0",
     "matplotlib>=3.7.2",
     "flake8>=4.0.1.1",
     "black>=24.4.2",

src/liger_kernel/transformers/__init__.py

@@ -22,7 +22,6 @@ from liger_kernel.transformers.monkey_patch import (  # noqa: F401
     apply_liger_kernel_to_qwen2,
     apply_liger_kernel_to_qwen2_vl,
 )
-from liger_kernel.transformers.orpo_trainer import LigerORPOTrainer  # noqa: F401
 from liger_kernel.transformers.rms_norm import LigerRMSNorm  # noqa: F401
 from liger_kernel.transformers.rope import liger_rotary_pos_emb  # noqa: F401
 from liger_kernel.transformers.swiglu import (  # noqa: F401

src/liger_kernel/transformers/trainer/__init__.py

@@ -0,0 +1,6 @@
+try:
+    from liger_kernel.transformers.trainer.orpo_trainer import (  # noqa: F401
+        LigerORPOTrainer,
+    )
+except ImportError:
+    raise ImportError("Please `pip install trl` to use LigerORPOTrainer")

src/liger_kernel/transformers/trainer/orpo_trainer.py

@@ -76,9 +76,7 @@ class LigerORPOTrainer(ORPOTrainer):
             padding_value=self.padding_value,
             device=self.accelerator.device,
         )
-        # if self.accelerator.is_main_process:
-        #     import pdb; pdb.set_trace()
-        # torch.distributed.barrier()
+
         model_kwargs = (
             {
                 "decoder_input_ids": self._shift_right(

test/convergence/test_mini_models_multimodal.py

@@ -16,7 +16,9 @@ from test.utils import (
 
 import pytest
 import torch
+import transformers
 from datasets import load_dataset
+from packaging import version
 from torch.utils.data import DataLoader
 from transformers import PreTrainedTokenizerFast
 
@@ -378,8 +380,9 @@ def run_mini_model_multimodal(
             5e-3,
             1e-5,
             marks=pytest.mark.skipif(
-                not QWEN2_VL_AVAILABLE,
-                reason="Qwen2-VL not available in this version of transformers",
+                not QWEN2_VL_AVAILABLE
+                or version.parse(transformers.__version__) >= version.parse("4.47.0"),
+                reason="Qwen2-VL not available in this version of transformers or transformers version >= 4.47.0",
             ),
         ),
         pytest.param(
@@ -398,8 +401,10 @@ def run_mini_model_multimodal(
                     not supports_bfloat16(), reason="bfloat16 not supported on this GPU"
                 ),
                 pytest.mark.skipif(
-                    not QWEN2_VL_AVAILABLE,
-                    reason="Qwen2-VL not available in this version of transformers",
+                    not QWEN2_VL_AVAILABLE
+                    or version.parse(transformers.__version__)
+                    >= version.parse("4.47.0"),
+                    reason="Qwen2-VL not available in this version of transformers or transformers version >= 4.47.0",
                 ),
             ],
         ),