mirror of
https://github.com/linkedin/Liger-Kernel.git
synced 2026-10-12 02:17:47 +00:00
Handle aux_loss for different transformer versions (#934)
## Summary <!--- This is a required section; please describe the main purpose of this proposed code change. ---> as only newer upcoming version of transformers exposes it. <!--- ## Details This is an optional section; is there anything specific that reviewers should be aware of? ---> ## Testing Done <!--- This is a required section; please describe how this change was tested. ---> <!-- Replace BLANK with your device type. For example, A100-80G-PCIe Complete the following tasks before sending your PR, and replace `[ ]` with `[x]` to indicate you have done them. --> - Hardware Type: <BLANK> - [x] run `make test` to ensure correctness - [x] run `make checkstyle` to ensure code style - [x] run `make test-convergence` to ensure convergence
This commit is contained in:
@@ -155,14 +155,18 @@ def lce_forward(
|
||||
output = output + (token_accuracy,) if token_accuracy is not None else output
|
||||
return output
|
||||
|
||||
# Return GLM4V MoE output with accuracy (using dict syntax to add extra field)
|
||||
return LigerGlm4vMoeCausalLMOutputWithPast(
|
||||
# Build output kwargs and include aux_loss only if present (depends on transformers version)
|
||||
output_kwargs = dict(
|
||||
loss=loss,
|
||||
logits=logits,
|
||||
past_key_values=outputs.past_key_values,
|
||||
hidden_states=outputs.hidden_states,
|
||||
attentions=outputs.attentions,
|
||||
rope_deltas=outputs.rope_deltas,
|
||||
aux_loss=outputs.aux_loss,
|
||||
token_accuracy=token_accuracy,
|
||||
)
|
||||
if hasattr(outputs, "aux_loss"):
|
||||
output_kwargs["aux_loss"] = outputs.aux_loss
|
||||
|
||||
# Return GLM4V MoE output with accuracy
|
||||
return LigerGlm4vMoeCausalLMOutputWithPast(**output_kwargs)
|
||||
|
||||
Reference in New Issue
Block a user