diff --git a/bigcode_eval/base.py b/bigcode_eval/base.py
index 8dffb98c1..9468b98cb 100644
--- a/bigcode_eval/base.py
+++ b/bigcode_eval/base.py
@@ -28,7 +28,8 @@ def __init__(self, stop_words=None, requires_execution=True):
             self.dataset = load_dataset(path=self.DATASET_PATH, name=self.DATASET_NAME)
         except Exception as e:
             warn(
-                f"Loading the dataset failed with {str(e)}. This task will use a locally downloaded dataset, not from the HF hub."
+                f"Loading the dataset failed with {str(e)}. This task will use a locally downloaded dataset, not from the HF hub. \
+                This is expected behavior for the DS-1000 benchmark but not for other benchmarks!"
             )
 
     @abstractmethod
@@ -92,4 +93,3 @@ def _stop_at_stop_token(decoded_string, stop_tokens):
             if stop_index != -1 and stop_index < min_stop_index:
                 min_stop_index = stop_index
         return decoded_string[:min_stop_index]
-