attempt combine_chunks

Signed-off-by: varun-edachali-dbx <varun.edachali@databricks.com>

Author: varun-edachali-dbx

src/databricks/sql/client.py

@@ -1435,6 +1435,8 @@ def _convert_arrow_table(self, table):
             date_as_object=True,
             timestamp_as_object=True,
         )
+        del table
+        del table_renamed
 
         res = df.to_numpy(na_value=None, dtype="object")
         return [ResultRow(*v) for v in res]
@@ -1465,6 +1467,7 @@ def fetchmany_arrow(self, size: int) -> "pyarrow.Table":
             results = pyarrow.concat_tables([results, partial_results])
             n_remaining_rows -= partial_results.num_rows
             self._next_row_index += partial_results.num_rows
+        results = results.combine_chunks()
 
         return results
 

src/databricks/sql/utils.py

@@ -273,6 +273,7 @@ def next_n_rows(self, num_rows: int) -> "pyarrow.Table":
                 self.table = self._create_next_table()
                 self.table_row_index = 0
             num_rows -= table_slice.num_rows
+        results = results.combine_chunks()
 
         logger.debug("CloudFetchQueue: collected {} next rows".format(results.num_rows))
         return results
@@ -296,6 +297,7 @@ def remaining_rows(self) -> "pyarrow.Table":
             self.table_row_index += table_slice.num_rows
             self.table = self._create_next_table()
             self.table_row_index = 0
+        results = results.combine_chunks()
         return results
 
     def _create_next_table(self) -> Union["pyarrow.Table", None]: