diff --git a/bigcode_eval/tasks/multiple.py b/bigcode_eval/tasks/multiple.py
index ec031c64e..7f514426d 100644
--- a/bigcode_eval/tasks/multiple.py
+++ b/bigcode_eval/tasks/multiple.py
@@ -90,7 +90,7 @@ def __init__(self, language):
             GeneralMultiPLE.DATASET_PATH,
             self.DATASET_NAME,
             revision=self.DATASET_REVISION)
-        stop_words = self.dataset["test"][0]["stop_tokens"]
+        stop_words = self.dataset["test"][0]["stop_tokens"] + ["<file_sep>"]
         super().__init__(
             stop_words=stop_words,
             requires_execution=True,