[tests] make cuda-only test work on other hardware accelerators (#3302)

* enable on xpu

* remove require_cuda

tests/test_modeling_utils.py

@@ -28,7 +28,6 @@
 from accelerate import init_empty_weights
 from accelerate.big_modeling import cpu_offload
 from accelerate.test_utils import (
-    require_cuda,
     require_huggingface_suite,
     require_multi_device,
     require_non_cpu,
@@ -853,7 +852,7 @@ def test_infer_auto_device_map_with_fallback_allocation_and_buffers(self):
         expected_device_map = {"batchnorm": 0, "linear1": "disk", "linear2": "disk"}
         assert device_map == expected_device_map
 
-    @require_cuda
+    @require_non_cpu
     def test_get_balanced_memory(self):
         model = ModelForTest()
         # model has size 236: linear1 64, batchnorm 72, linear2 100