fix: bad performance when walking the file tree and check includes (#…

…4778)
bentoml · Jun 6, 2024 · 8abc7ad · 8abc7ad
1 parent 8bfd059
commit 8abc7ad
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 10 deletions.
diff --git a/src/bentoml/_internal/bento/bento.py b/src/bentoml/_internal/bento/bento.py
@@ -257,15 +257,15 @@ def create(
             )
         bento_fs.makedir(BENTO_PROJECT_DIR_NAME)
         target_fs = bento_fs.opendir(BENTO_PROJECT_DIR_NAME)
+        ignore_specs = list(specs.from_path(build_ctx))
 
         for dir_path, _, files in ctx_fs.walk():
             for f in files:
                 path = fs.path.combine(dir_path, f.name).lstrip("/")
                 if specs.includes(
                     path,
                     recurse_exclude_spec=filter(
-                        lambda s: fs.path.isparent(s[0], dir_path),
-                        specs.from_path(build_ctx),
+                        lambda s: fs.path.isparent(s[0], dir_path), ignore_specs
                     ),
                 ):
                     if ctx_fs.getsize(path) > 10 * 1024 * 1024:

diff --git a/src/bentoml/_internal/bento/build_config.py b/src/bentoml/_internal/bento/build_config.py
@@ -949,21 +949,20 @@ def includes(
             and not self._exclude.match_file(path)
             and not self.extra.match_file(path)
         )
-        if to_include:
-            if recurse_exclude_spec is not None:
-                return not any(
-                    ignore_spec.match_file(fs.path.relativefrom(ignore_parent, path))
-                    for ignore_parent, ignore_spec in recurse_exclude_spec
-                )
-        return False
+        if to_include and recurse_exclude_spec is not None:
+            return not any(
+                ignore_spec.match_file(fs.path.relativefrom(ignore_parent, path))
+                for ignore_parent, ignore_spec in recurse_exclude_spec
+            )
+        return to_include
 
     def from_path(self, path: str) -> t.Generator[t.Tuple[str, PathSpec], None, None]:
         """
         yield (parent, exclude_spec) from .bentoignore file of a given path
         """
         fs_ = fs.open_fs(path)
         for file in fs_.walk.files(filter=[".bentoignore"]):
-            dir_path = "".join(fs.path.parts(file)[:-1])
+            dir_path = fs.path.dirname(file)
             yield dir_path, PathSpec.from_lines("gitwildmatch", fs_.open(file))