integration-tests: Ignore files larger than 10MB

delb-xml · Sep 13, 2024 · ebbe298 · ebbe298
1 parent 6e66277
commit ebbe298
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/integration-tests/fetch-corpora.py b/integration-tests/fetch-corpora.py
@@ -54,6 +54,7 @@ class Archive(NamedTuple):
 
 CORPORA_PATH: Final = Path(__file__).parent.resolve() / "corpora"
 
+FILE_SIZE_LIMIT: Final = 10 * 1024 ** 2
 
 ARCHIVE_DESCRIPTIONS: Final = (
     Archive(
@@ -286,6 +287,9 @@ def _filter(member: tarfile.TarInfo, path: str) -> tarfile.TarInfo | None:
         if member is None:
             return None
 
+        if member.size > FILE_SIZE_LIMIT:
+            return None
+
         member_path = member.name
         root_folder = archive_description.archive_documents_root
         if member_path.endswith(".xml") and member_path.startswith(root_folder):