PERF-modin-project#5675: make index calculation for read_csv functi…

…on lazy; introduce `ModinIndex` (modin-project#5677) Co-authored-by: Iaroslav Igoshev <[email protected]> Co-authored-by: Dmitry Chigarev <[email protected]> Signed-off-by: Anatoly Myachev <[email protected]>
labanyamukhopadhyay · Mar 30, 2023 · edccfd0 · edccfd0
1 parent 8130923
commit edccfd0
Show file tree

Hide file tree

Showing 15 changed files with 387 additions and 119 deletions.
diff --git a/.github/workflows/ci-notebooks.yml b/.github/workflows/ci-notebooks.yml
@@ -65,7 +65,7 @@ jobs:
           pip install git+https://github.com/modin-project/modin-spreadsheet.git@49ffd89f683f54c311867d602c55443fb11bf2a5
         if: matrix.execution != 'hdk_on_native'
       # Build Modin from sources for `hdk_on_native`
-      - run: pip install .
+      - run: pip install -e .
         if: matrix.execution == 'hdk_on_native'
       # install test dependencies
       # NOTE: If you are changing the set of packages installed here, make sure that

diff --git a/modin/core/dataframe/algebra/binary.py b/modin/core/dataframe/algebra/binary.py
@@ -197,7 +197,6 @@ def caller(
             """
             axis = kwargs.get("axis", 0)
             shape_hint = None
-            self_columns = query_compiler._modin_frame._columns_cache
             if isinstance(other, type(query_compiler)):
                 if broadcast:
                     assert (
@@ -211,13 +210,13 @@ def caller(
                         other = other.transpose()
 
                     if (
-                        self_columns is not None
-                        and other._modin_frame._columns_cache is not None
+                        query_compiler._modin_frame.has_materialized_columns
+                        and other._modin_frame.has_materialized_columns
                     ):
                         if (
-                            len(self_columns) == 1
+                            len(query_compiler.columns) == 1
                             and len(other.columns) == 1
-                            and self_columns.equals(other.columns)
+                            and query_compiler.columns.equals(other.columns)
                         ):
                             shape_hint = "column"
                     return query_compiler.__constructor__(
@@ -246,11 +245,11 @@ def caller(
                             dtypes = compute_dtypes_common_cast(query_compiler, other)
                             dtypes = dtypes.apply(coerce_int_to_float64)
                     if (
-                        self_columns is not None
-                        and other._modin_frame._columns_cache is not None
+                        query_compiler._modin_frame.has_materialized_columns
+                        and other._modin_frame.has_materialized_columns
                     ):
                         if (
-                            len(self_columns) == 1
+                            len(query_compiler.columns) == 1
                             and len(other.columns) == 1
                             and query_compiler.columns.equals(other.columns)
                         ):
@@ -277,8 +276,8 @@ def caller(
                     )
                 else:
                     if (
-                        self_columns is not None
-                        and len(self_columns) == 1
+                        query_compiler._modin_frame.has_materialized_columns
+                        and len(query_compiler._modin_frame.columns) == 1
                         and is_scalar(other)
                     ):
                         shape_hint = "column"