Merge branch 'main' into qwen2_5_vl

linkedin · Feb 8, 2025 · c802223 · c802223
2 parents f60c121 + bcde83c
commit c802223
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -309,6 +309,7 @@ loss.backward()
 
 ## Sponsorship and Collaboration
 
+- [Glows.ai](https://platform.glows.ai/): Sponsoring NVIDIA GPUs for our open source developers.
 - [AMD](https://www.amd.com/en.html): Providing AMD GPUs for our AMD CI.
 - [Intel](https://www.intel.com/): Providing Intel GPUs for our Intel CI.
 - [Modal](https://modal.com/): Free 3000 credits from GPU MODE IRL for our NVIDIA CI.

diff --git a/src/liger_kernel/chunked_loss/dpo_loss.py b/src/liger_kernel/chunked_loss/dpo_loss.py
@@ -45,9 +45,12 @@ def preference_loss_fn(
         chosen_logratios = chosen_logps - ref_chosen_logps
         rejected_logratios = rejected_logps - ref_rejected_logps
 
+        chosen_rewards = beta * (chosen_logps - ref_chosen_logps)
+        rejected_rewards = beta * (rejected_logps - ref_rejected_logps)
+
         logits_diff = beta * (chosen_logratios - rejected_logratios)
         loss = -F.logsigmoid(logits_diff).sum() / (full_target.shape[0] // 2)
-        return loss
+        return loss, chosen_rewards, rejected_rewards
 
     @staticmethod
     def forward(
@@ -99,7 +102,7 @@ def __init__(
         beta: float = 0.1,
         compute_nll_loss: bool = False,
         compiled: bool = True,
-        use_ref_model: bool = False,
+        use_ref_model: bool = True,
     ):
         """
         Args:

diff --git a/test/chunked_loss/test_dpo_loss.py b/test/chunked_loss/test_dpo_loss.py
@@ -56,9 +56,12 @@ def alignment_loss(
         chosen_logratios = policy_chosen_logps - ref_chosen_logps
         rejected_logratios = policy_rejected_logps - ref_rejected_logps
 
+        chosen_rewards = self.beta * (policy_chosen_logps - ref_chosen_logps)   
+        rejected_rewards = self.beta * (policy_rejected_logps - ref_rejected_logps)
+
         logits_diff = self.beta * (chosen_logratios - rejected_logratios)
         losses = -F.logsigmoid(logits_diff)
-        return losses
+        return losses, chosen_rewards, rejected_rewards
 
 
 class TorchLMHeadDPO(torch.nn.Module):