diff --git a/LobbyCompatibility/Behaviours/ButtonEventHandler.cs b/LobbyCompatibility/Behaviours/ButtonEventHandler.cs
index cf42f43..f447996 100644
--- a/LobbyCompatibility/Behaviours/ButtonEventHandler.cs
+++ b/LobbyCompatibility/Behaviours/ButtonEventHandler.cs
@@ -65,6 +65,18 @@ public void SetButtonImageData(Image image, Sprite normalSprite, Sprite highligh
         _image = image;
         _normalSprite = normalSprite;
         _highlightedSprite = highlightedSprite;
+
+        SetColor(normalColor, highlightedColor);
+        SetHighlighted(false);
+    }
+
+    /// <summary>
+    ///     Sets the button's color data
+    /// </summary>
+    /// <param name="normalColor"> Color to use when the button is not highlighted </param>
+    /// <param name="highlightedColor"> Color to use when the button is highlighted </param>
+    public void SetColor(Color normalColor, Color highlightedColor)
+    {
         _normalColor = normalColor;
         _highlightedColor = highlightedColor;
 
diff --git a/LobbyCompatibility/Behaviours/ModdedLobbySlot.cs b/LobbyCompatibility/Behaviours/ModdedLobbySlot.cs
index 174e4e3..7b19bd5 100644
--- a/LobbyCompatibility/Behaviours/ModdedLobbySlot.cs
+++ b/LobbyCompatibility/Behaviours/ModdedLobbySlot.cs
@@ -7,6 +7,7 @@
 using UnityEngine.UI;
 using Image = UnityEngine.UI.Image;
 using Color = UnityEngine.Color;
+using System.Linq;
 
 namespace LobbyCompatibility.Behaviours;
 
@@ -18,6 +19,7 @@ public class ModdedLobbySlot : MonoBehaviour
 {
     private List<ButtonEventHandler> _buttonEventHandlers = new();
     private RectTransform? _buttonTransform;
+    private Button? _joinButton;
     private LobbyDiff? _lobbyDiff;
     private LobbySlot? _lobbySlot;
     private Transform? _parentContainer;
@@ -36,8 +38,8 @@ internal void Setup(LobbySlot lobbySlot)
             return;
 
         // Find "Join Lobby" button template
-        var joinButton = GetComponentInChildren<Button>();
-        if (joinButton == null)
+        _joinButton = GetComponentInChildren<Button>();
+        if (_joinButton == null)
             return;
 
         // Get button sprites (depending on the lobby type/status, sometimes it will need to be a warning/alert for incompatible lobbies)
@@ -51,17 +53,28 @@ internal void Setup(LobbySlot lobbySlot)
             playerCount.transform.localPosition = new Vector3(32f, localPosition.y, localPosition.z);
 
             // Create the actual modlist button to the left of the player count text
-            CreateModListButton(joinButton, sprite, invertedSprite, _lobbySlot.LobbyName.color, playerCount.transform);
+            CreateModListButton(_joinButton, sprite, invertedSprite, _lobbySlot.LobbyName.color, playerCount.transform);
         }
+    }
+
+    // Join button isn't initialized until Awake, so we need to wait until then to modify join button data
+    private void Start()
+    {
+        var buttonText = _joinButton?.GetComponentInChildren<TextMeshProUGUI>();
+        if (_lobbyDiff == null || _joinButton == null || buttonText == null)
+            return;
+
+        if (_lobbyDiff.GetModdedLobbyType() != LobbyDiffResult.Incompatible)
+            return;
 
         // If lobby is incompatible, disable the join button (because it won't work)
-        if (_lobbyDiff.GetModdedLobbyType() == LobbyDiffResult.Incompatible)
-        {
-            joinButton.interactable = false;
+        _joinButton.interactable = false;
 
-            // Turn joinButton into a modlist button, so we can get the modlist on hover
-            SetupModListButtonEvents(joinButton);
-        }
+        // If needed, setup brighter button color based on the now-enabled button text
+        _buttonEventHandlers.ForEach(handler => handler.SetColor(buttonText.color, buttonText.color));
+
+        // Turn joinButton into a modlist button, so we can get the modlist on hover
+        SetupModListButtonEvents(_joinButton);
     }
 
     /// <summary>
@@ -131,14 +144,9 @@ public void SetParentContainer(Transform parentTransform)
         SetupButtonPositioning(_buttonTransform);
         SetupButtonImage(buttonImageTransform, buttonImage);
 
-        // Get brighter color for incompatible lobbies
-        var incompatibleColor = buttonText?.color ?? color;
-
         // Inject custom event handling
         var buttonEventHandler = SetupModListButtonEvents(button);
-        buttonEventHandler.SetButtonImageData(buttonImage, sprite, invertedSprite,
-            _lobbyDiff.GetModdedLobbyType() == LobbyDiffResult.Compatible ? color : incompatibleColor,
-            _lobbyDiff.GetModdedLobbyType() == LobbyDiffResult.Compatible ? color : incompatibleColor);
+        buttonEventHandler.SetButtonImageData(buttonImage, sprite, invertedSprite, color, color);
 
         return button;
     }