add args to to_pandas and update cdf path too

Author: PatrickJin-db

python/delta_sharing/reader.py

@@ -23,7 +23,6 @@
 import fsspec
 import os
 import pandas as pd
-import pyarrow as pa
 import tempfile
 from pyarrow.dataset import dataset
 
@@ -131,14 +130,9 @@ def __to_pandas_kernel(self):
             return pd.DataFrame(columns=schema.names)
 
         batches = scan.execute(interface)
-        pdfs = [batch.to_pandas() for batch in batches]
-        
-        result = pd.concat(
-            pdfs,
-            axis=0,
-            ignore_index=True,
-            copy=False,
-        )
+        pdfs = [batch.to_pandas(self_destruct=True) for batch in batches]
+
+        result = pd.concat(pdfs, axis=0, ignore_index=True, copy=False)
 
         # Apply residual limit that was not handled from server pushdown
         result = result.head(self._limit)
@@ -372,8 +366,8 @@ def __table_changes_to_pandas_kernel(self, cdfOptions: CdfOptions) -> pd.DataFra
                 schema = scan.execute(interface).schema
                 result = pd.DataFrame(columns=schema.names)
             else:
-                table = pa.Table.from_batches(scan.execute(interface))
-                result = table.to_pandas()
+                pdfs = [batch.to_pandas(self_destruct=True) for batch in scan.execute(interface)]
+                result = pd.concat(pdfs, axis=0, ignore_index=True, copy=False)
         finally:
             # Delete the temp folder explicitly and remove the delta format from header
             temp_dir.cleanup()