diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index 15c58b894c0..176e15a5a31 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -14,6 +14,7 @@ use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::TypedArrayRef; use vortex_array::arrays::VariantArray; use vortex_array::arrow::ArrowArrayExecutor; @@ -183,17 +184,20 @@ impl ParquetVariantData { } }) .unwrap_or(Validity::NonNullable); - let metadata = - ArrayRef::from_arrow(arrow_variant.metadata_field() as &dyn ArrowArray, false)?; + let metadata = ArrayRef::from_arrow_in( + arrow_variant.metadata_field() as &dyn ArrowArray, + false, + &LEGACY_SESSION, + )?; let value = arrow_variant .value_field() - .map(|v| ArrayRef::from_arrow(v as &dyn ArrowArray, value_nullable)) + .map(|v| ArrayRef::from_arrow_in(v as &dyn ArrowArray, value_nullable, &LEGACY_SESSION)) .transpose()?; let typed_value = arrow_variant .typed_value_field() - .map(|tv| ArrayRef::from_arrow(tv.as_ref(), typed_value_nullable)) + .map(|tv| ArrayRef::from_arrow_in(tv.as_ref(), typed_value_nullable, &LEGACY_SESSION)) .transpose()?; let pv = ParquetVariant::try_new(validity, metadata, value, typed_value)?; diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index 63c9f9857fe..976b4dfbea3 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -130,7 +130,7 @@ pub fn vortex_runend::RunEndData::array_hash(&self, &mut impl vortex_array::arrow::FromArrowArray<&arrow_array::array::run_array::RunArray> for vortex_runend::RunEndData where ::Native: vortex_array::dtype::ptype::NativePType -pub fn vortex_runend::RunEndData::from_arrow(&arrow_array::array::run_array::RunArray, bool) -> vortex_error::VortexResult +pub fn vortex_runend::RunEndData::from_arrow_in(&arrow_array::array::run_array::RunArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult pub struct vortex_runend::RunEndDataParts diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 0b1d7c2ca3e..82eb2b99092 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -17,6 +17,7 @@ use vortex_array::search_sorted::SearchSortedSide; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; +use vortex_session::VortexSession; use crate::RunEndData; use crate::ops::find_slice_end_index; @@ -25,14 +26,18 @@ impl FromArrowArray<&RunArray> for RunEndData where R::Native: NativePType, { - fn from_arrow(array: &RunArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + array: &RunArray, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { let offset = array.run_ends().offset(); let len = array.run_ends().len(); let ends_buf = Buffer::::from_arrow_scalar_buffer(array.run_ends().inner().clone()); let ends = PrimitiveArray::new(ends_buf, Validity::NonNullable) .reinterpret_cast(R::Native::PTYPE.to_unsigned()); - let values = ArrayRef::from_arrow(array.values().as_ref(), nullable)?; + let values = ArrayRef::from_arrow_in(array.values().as_ref(), nullable, session)?; let ends_array = PrimitiveArray::from_buffer_handle( ends.buffer_handle().clone(), @@ -125,7 +130,7 @@ mod tests { Buffer::::from_arrow_scalar_buffer(array.run_ends().inner().clone()); let ends = PrimitiveArray::new(ends_buf, Validity::NonNullable) .reinterpret_cast(R::Native::PTYPE.to_unsigned()); - let values = ArrayRef::from_arrow(array.values().as_ref(), nullable)?; + let values = ArrayRef::from_arrow_in(array.values().as_ref(), nullable, &SESSION)?; let ends_array = PrimitiveArray::from_buffer_handle( ends.buffer_handle().clone(), diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index b6de5284907..ed8f0d80486 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -7144,157 +7144,233 @@ pub fn vortex_array::ArrayRef::execute_record_batch(self, &arrow_schema::schema: pub fn vortex_array::ArrayRef::execute_record_batches(self, &arrow_schema::schema::Schema, &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub trait vortex_array::arrow::FromArrowArray +pub trait vortex_array::arrow::FromArrowArray: core::marker::Sized -pub fn vortex_array::arrow::FromArrowArray::from_arrow(A, bool) -> vortex_error::VortexResult where Self: core::marker::Sized +pub fn vortex_array::arrow::FromArrowArray::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::arrow::FromArrowArray::from_arrow_in(A, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::boolean_array::BooleanArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::boolean_array::BooleanArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::boolean_array::BooleanArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::fixed_size_list_array::FixedSizeListArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::fixed_size_list_array::FixedSizeListArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::fixed_size_list_array::FixedSizeListArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::null_array::NullArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::null_array::NullArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::null_array::NullArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::struct_array::StructArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::struct_array::StructArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::struct_array::StructArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::record_batch::RecordBatch> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::record_batch::RecordBatch, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::record_batch::RecordBatch, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&dyn arrow_array::array::Array> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&dyn arrow_array::array::Array, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&dyn arrow_array::array::Array, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(arrow_array::record_batch::RecordBatch, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(arrow_array::record_batch::RecordBatch, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::dictionary_array::DictionaryArray> for vortex_array::arrays::dict::DictArray -pub fn vortex_array::arrays::dict::DictArray::from_arrow(&arrow_array::array::dictionary_array::DictionaryArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::DictArray::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::DictArray::from_arrow_in(&arrow_array::array::dictionary_array::DictionaryArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::list_view_array::GenericListViewArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::list_view_array::GenericListViewArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::list_view_array::GenericListViewArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::list_array::GenericListArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::list_array::GenericListArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::list_array::GenericListArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::byte_array::GenericByteArray> for vortex_array::ArrayRef where ::Offset: vortex_array::dtype::IntegerPType -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::byte_array::GenericByteArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::byte_array::GenericByteArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::byte_view_array::GenericByteViewArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::byte_view_array::GenericByteViewArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::byte_view_array::GenericByteViewArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult pub trait vortex_array::arrow::IntoArrowArray @@ -8816,25 +8892,37 @@ pub trait vortex_array::dtype::arrow::FromArrowType: core::marker::Sized pub fn vortex_array::dtype::arrow::FromArrowType::from_arrow(T) -> Self +pub fn vortex_array::dtype::arrow::FromArrowType::from_arrow_in(T, &vortex_session::VortexSession) -> Self + impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::field::Field> for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::from_arrow(&arrow_schema::field::Field) -> Self +pub fn vortex_array::dtype::DType::from_arrow(T) -> Self + +pub fn vortex_array::dtype::DType::from_arrow_in(&arrow_schema::field::Field, &vortex_session::VortexSession) -> Self impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::fields::Fields> for vortex_array::dtype::StructFields -pub fn vortex_array::dtype::StructFields::from_arrow(&arrow_schema::fields::Fields) -> Self +pub fn vortex_array::dtype::StructFields::from_arrow(T) -> Self + +pub fn vortex_array::dtype::StructFields::from_arrow_in(&arrow_schema::fields::Fields, &vortex_session::VortexSession) -> Self impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::schema::Schema> for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::from_arrow(&arrow_schema::schema::Schema) -> Self +pub fn vortex_array::dtype::DType::from_arrow(T) -> Self + +pub fn vortex_array::dtype::DType::from_arrow_in(&arrow_schema::schema::Schema, &vortex_session::VortexSession) -> Self impl vortex_array::dtype::arrow::FromArrowType<(&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability)> for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::from_arrow((&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability)) -> Self +pub fn vortex_array::dtype::DType::from_arrow(T) -> Self + +pub fn vortex_array::dtype::DType::from_arrow_in((&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability), &vortex_session::VortexSession) -> Self impl vortex_array::dtype::arrow::FromArrowType> for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::from_arrow(arrow_schema::schema::SchemaRef) -> Self +pub fn vortex_array::dtype::DType::from_arrow(T) -> Self + +pub fn vortex_array::dtype::DType::from_arrow_in(arrow_schema::schema::SchemaRef, &vortex_session::VortexSession) -> Self pub trait vortex_array::dtype::arrow::TryFromArrowType: core::marker::Sized @@ -9374,19 +9462,27 @@ pub fn vortex_array::dtype::DType::hash<__H: core::hash::Hasher>(&self, &mut __H impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::field::Field> for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::from_arrow(&arrow_schema::field::Field) -> Self +pub fn vortex_array::dtype::DType::from_arrow(T) -> Self + +pub fn vortex_array::dtype::DType::from_arrow_in(&arrow_schema::field::Field, &vortex_session::VortexSession) -> Self impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::schema::Schema> for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::from_arrow(&arrow_schema::schema::Schema) -> Self +pub fn vortex_array::dtype::DType::from_arrow(T) -> Self + +pub fn vortex_array::dtype::DType::from_arrow_in(&arrow_schema::schema::Schema, &vortex_session::VortexSession) -> Self impl vortex_array::dtype::arrow::FromArrowType<(&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability)> for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::from_arrow((&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability)) -> Self +pub fn vortex_array::dtype::DType::from_arrow(T) -> Self + +pub fn vortex_array::dtype::DType::from_arrow_in((&arrow_schema::datatype::DataType, vortex_array::dtype::Nullability), &vortex_session::VortexSession) -> Self impl vortex_array::dtype::arrow::FromArrowType> for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::from_arrow(arrow_schema::schema::SchemaRef) -> Self +pub fn vortex_array::dtype::DType::from_arrow(T) -> Self + +pub fn vortex_array::dtype::DType::from_arrow_in(arrow_schema::schema::SchemaRef, &vortex_session::VortexSession) -> Self impl vortex_flatbuffers::FlatBufferRoot for vortex_array::dtype::DType @@ -10340,7 +10436,9 @@ pub fn vortex_array::dtype::StructFields::hash<__H: core::hash::Hasher>(&self, & impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::fields::Fields> for vortex_array::dtype::StructFields -pub fn vortex_array::dtype::StructFields::from_arrow(&arrow_schema::fields::Fields) -> Self +pub fn vortex_array::dtype::StructFields::from_arrow(T) -> Self + +pub fn vortex_array::dtype::StructFields::from_arrow_in(&arrow_schema::fields::Fields, &vortex_session::VortexSession) -> Self impl core::iter::traits::collect::FromIterator<(T, V)> for vortex_array::dtype::StructFields where T: core::convert::Into, V: core::convert::Into @@ -22196,6 +22294,8 @@ impl vortex_array::Array pub fn vortex_array::Array::into_record_batch_with_schema(self, impl core::convert::AsRef) -> vortex_error::VortexResult +pub fn vortex_array::Array::into_record_batch_with_schema_in(self, impl core::convert::AsRef, &vortex_session::VortexSession) -> vortex_error::VortexResult + impl vortex_array::Array pub fn vortex_array::Array::from_bytes(alloc::vec::Vec<&[u8]>) -> Self @@ -22724,131 +22824,195 @@ pub fn vortex_array::ArrayRef::execute_record_batches(self, &arrow_schema::schem impl vortex_array::arrow::FromArrowArray<&arrow_array::array::boolean_array::BooleanArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::boolean_array::BooleanArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::boolean_array::BooleanArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::fixed_size_list_array::FixedSizeListArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::fixed_size_list_array::FixedSizeListArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::fixed_size_list_array::FixedSizeListArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::null_array::NullArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::null_array::NullArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::null_array::NullArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::primitive_array::PrimitiveArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::primitive_array::PrimitiveArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::primitive_array::PrimitiveArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::struct_array::StructArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::struct_array::StructArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::struct_array::StructArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::record_batch::RecordBatch> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::record_batch::RecordBatch, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::record_batch::RecordBatch, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&dyn arrow_array::array::Array> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&dyn arrow_array::array::Array, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&dyn arrow_array::array::Array, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(arrow_array::record_batch::RecordBatch, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(arrow_array::record_batch::RecordBatch, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::IntoArrowArray for vortex_array::ArrayRef @@ -22916,19 +23080,27 @@ pub fn vortex_array::ArrayRef::index_lt(&self, usize, &V) -> vortex_error::Vorte impl vortex_array::arrow::FromArrowArray<&arrow_array::array::list_view_array::GenericListViewArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::list_view_array::GenericListViewArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::list_view_array::GenericListViewArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::list_array::GenericListArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::list_array::GenericListArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::list_array::GenericListArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::byte_array::GenericByteArray> for vortex_array::ArrayRef where ::Offset: vortex_array::dtype::IntegerPType -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::byte_array::GenericByteArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::byte_array::GenericByteArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::arrow::FromArrowArray<&arrow_array::array::byte_view_array::GenericByteViewArray> for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from_arrow(&arrow_array::array::byte_view_array::GenericByteViewArray, bool) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::from_arrow(A, bool) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::from_arrow_in(&arrow_array::array::byte_view_array::GenericByteViewArray, bool, &vortex_session::VortexSession) -> vortex_error::VortexResult impl core::convert::AsRef for vortex_array::Array diff --git a/vortex-array/src/arrays/filter/execute/varbinview.rs b/vortex-array/src/arrays/filter/execute/varbinview.rs index 6c0bb052bc2..f1b5572898a 100644 --- a/vortex-array/src/arrays/filter/execute/varbinview.rs +++ b/vortex-array/src/arrays/filter/execute/varbinview.rs @@ -37,7 +37,11 @@ fn arrow_filter_fn(array: &ArrayRef, mask: &Mask) -> vortex_error::VortexResult< let mask_array = BooleanArray::new(values.bit_buffer().clone().into(), None); let filtered = arrow_select::filter::filter(array_ref.as_ref(), &mask_array)?; - ArrayRef::from_arrow(filtered.as_ref(), array.dtype().is_nullable()) + ArrayRef::from_arrow_in( + filtered.as_ref(), + array.dtype().is_nullable(), + &LEGACY_SESSION, + ) } #[cfg(test)] diff --git a/vortex-array/src/arrow/convert.rs b/vortex-array/src/arrow/convert.rs index 6eafa6033c5..39fa57c51ea 100644 --- a/vortex-array/src/arrow/convert.rs +++ b/vortex-array/src/arrow/convert.rs @@ -66,6 +66,7 @@ use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; +use vortex_session::VortexSession; use crate::ArrayRef; use crate::IntoArray; @@ -138,7 +139,11 @@ where macro_rules! impl_from_arrow_primitive { ($T:path) => { impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef { - fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> VortexResult { + fn from_arrow_in( + value: &ArrowPrimitiveArray<$T>, + nullable: bool, + _session: &VortexSession, + ) -> VortexResult { let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone()); let validity = nulls(value.nulls(), nullable); Ok(PrimitiveArray::new(buffer, validity).into_array()) @@ -160,9 +165,10 @@ impl_from_arrow_primitive!(Float32Type); impl_from_arrow_primitive!(Float64Type); impl FromArrowArray<&ArrowPrimitiveArray> for ArrayRef { - fn from_arrow( + fn from_arrow_in( array: &ArrowPrimitiveArray, nullable: bool, + _session: &VortexSession, ) -> VortexResult { let decimal_type = DecimalDType::new(array.precision(), array.scale()); let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone()); @@ -172,9 +178,10 @@ impl FromArrowArray<&ArrowPrimitiveArray> for ArrayRef { } impl FromArrowArray<&ArrowPrimitiveArray> for ArrayRef { - fn from_arrow( + fn from_arrow_in( array: &ArrowPrimitiveArray, nullable: bool, + _session: &VortexSession, ) -> VortexResult { let decimal_type = DecimalDType::new(array.precision(), array.scale()); let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone()); @@ -184,9 +191,10 @@ impl FromArrowArray<&ArrowPrimitiveArray> for ArrayRef { } impl FromArrowArray<&ArrowPrimitiveArray> for ArrayRef { - fn from_arrow( + fn from_arrow_in( array: &ArrowPrimitiveArray, nullable: bool, + _session: &VortexSession, ) -> VortexResult { let decimal_type = DecimalDType::new(array.precision(), array.scale()); let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone()); @@ -196,9 +204,10 @@ impl FromArrowArray<&ArrowPrimitiveArray> for ArrayRef { } impl FromArrowArray<&ArrowPrimitiveArray> for ArrayRef { - fn from_arrow( + fn from_arrow_in( array: &ArrowPrimitiveArray, nullable: bool, + _session: &VortexSession, ) -> VortexResult { let decimal_type = DecimalDType::new(array.precision(), array.scale()); let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone()); @@ -215,9 +224,10 @@ impl FromArrowArray<&ArrowPrimitiveArray> for ArrayRef { macro_rules! impl_from_arrow_temporal { ($T:path) => { impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef { - fn from_arrow( + fn from_arrow_in( value: &ArrowPrimitiveArray<$T>, nullable: bool, + _session: &VortexSession, ) -> vortex_error::VortexResult { Ok(temporal_array(value, nullable)) } @@ -269,7 +279,11 @@ impl FromArrowArray<&GenericByteArray> for ArrayRef where ::Offset: IntegerPType, { - fn from_arrow(value: &GenericByteArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + value: &GenericByteArray, + nullable: bool, + _session: &VortexSession, + ) -> VortexResult { let dtype = match T::DATA_TYPE { DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()), DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()), @@ -289,7 +303,11 @@ where } impl FromArrowArray<&GenericByteViewArray> for ArrayRef { - fn from_arrow(value: &GenericByteViewArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + value: &GenericByteViewArray, + nullable: bool, + _session: &VortexSession, + ) -> VortexResult { let dtype = match T::DATA_TYPE { DataType::BinaryView => DType::Binary(nullable.into()), DataType::Utf8View => DType::Utf8(nullable.into()), @@ -321,7 +339,11 @@ impl FromArrowArray<&GenericByteViewArray> for ArrayRef { } impl FromArrowArray<&ArrowBooleanArray> for ArrayRef { - fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + value: &ArrowBooleanArray, + nullable: bool, + _session: &VortexSession, + ) -> VortexResult { Ok(BoolArray::new( value.values().clone().into(), nulls(value.nulls(), nullable), @@ -379,7 +401,11 @@ fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData { } impl FromArrowArray<&ArrowStructArray> for ArrayRef { - fn from_arrow(value: &ArrowStructArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + value: &ArrowStructArray, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { Ok(StructArray::try_new( value.column_names().iter().copied().collect(), value @@ -391,9 +417,9 @@ impl FromArrowArray<&ArrowStructArray> for ArrayRef { // out here because Vortex is a little more strict. if c.null_count() > 0 && !field.is_nullable() { let stripped = make_array(remove_nulls(c.into_data())); - Self::from_arrow(stripped.as_ref(), false) + Self::from_arrow_in(stripped.as_ref(), false, session) } else { - Self::from_arrow(c.as_ref(), field.is_nullable()) + Self::from_arrow_in(c.as_ref(), field.is_nullable(), session) } }) .collect::>>()?, @@ -405,7 +431,11 @@ impl FromArrowArray<&ArrowStructArray> for ArrayRef { } impl FromArrowArray<&GenericListArray> for ArrayRef { - fn from_arrow(value: &GenericListArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + value: &GenericListArray, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { // Extract the validity of the underlying element array. let elements_are_nullable = match value.data_type() { DataType::List(field) => field.is_nullable(), @@ -413,7 +443,8 @@ impl FromArrowArray<&GenericListArray> for dt => vortex_panic!("Invalid data type for ListArray: {dt}"), }; - let elements = Self::from_arrow(value.values().as_ref(), elements_are_nullable)?; + let elements = + Self::from_arrow_in(value.values().as_ref(), elements_are_nullable, session)?; // `offsets` are always non-nullable. let offsets = value.offsets().clone().into_array(); @@ -424,7 +455,11 @@ impl FromArrowArray<&GenericListArray> for } impl FromArrowArray<&GenericListViewArray> for ArrayRef { - fn from_arrow(array: &GenericListViewArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + array: &GenericListViewArray, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { // Extract the validity of the underlying element array. let elements_are_nullable = match array.data_type() { DataType::ListView(field) => field.is_nullable(), @@ -432,7 +467,8 @@ impl FromArrowArray<&GenericListViewArray> dt => vortex_panic!("Invalid data type for ListViewArray: {dt}"), }; - let elements = Self::from_arrow(array.values().as_ref(), elements_are_nullable)?; + let elements = + Self::from_arrow_in(array.values().as_ref(), elements_are_nullable, session)?; // `offsets` and `sizes` are always non-nullable. let offsets = array.offsets().clone().into_array(); @@ -444,13 +480,17 @@ impl FromArrowArray<&GenericListViewArray> } impl FromArrowArray<&ArrowFixedSizeListArray> for ArrayRef { - fn from_arrow(array: &ArrowFixedSizeListArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + array: &ArrowFixedSizeListArray, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { let DataType::FixedSizeList(field, list_size) = array.data_type() else { vortex_panic!("Invalid data type for ListArray: {}", array.data_type()); }; Ok(FixedSizeListArray::try_new( - Self::from_arrow(array.values().as_ref(), field.is_nullable())?, + Self::from_arrow_in(array.values().as_ref(), field.is_nullable(), session)?, *list_size as u32, nulls(array.nulls(), nullable), array.len(), @@ -460,17 +500,25 @@ impl FromArrowArray<&ArrowFixedSizeListArray> for ArrayRef { } impl FromArrowArray<&ArrowNullArray> for ArrayRef { - fn from_arrow(value: &ArrowNullArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + value: &ArrowNullArray, + nullable: bool, + _session: &VortexSession, + ) -> VortexResult { assert!(nullable); Ok(NullArray::new(value.len()).into_array()) } } impl FromArrowArray<&DictionaryArray> for DictArray { - fn from_arrow(array: &DictionaryArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + array: &DictionaryArray, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { let keys = AnyDictionaryArray::keys(array); - let keys = ArrayRef::from_arrow(keys, keys.is_nullable())?; - let values = ArrayRef::from_arrow(array.values().as_ref(), nullable)?; + let keys = ArrayRef::from_arrow_in(keys, keys.is_nullable(), session)?; + let values = ArrayRef::from_arrow_in(array.values().as_ref(), nullable, session)?; // SAFETY: we assume that Arrow has checked the invariants on construction. Ok(unsafe { DictArray::new_unchecked(keys, values) }) } @@ -494,118 +542,180 @@ fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity { } impl FromArrowArray<&dyn ArrowArray> for ArrayRef { - fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> VortexResult { + fn from_arrow_in( + array: &dyn ArrowArray, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { match array.data_type() { - DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable), - DataType::UInt8 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::UInt16 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::UInt32 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::UInt64 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Int8 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Int16 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Int32 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Int64 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Float16 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Float32 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Float64 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Utf8 => Self::from_arrow(array.as_string::(), nullable), - DataType::LargeUtf8 => Self::from_arrow(array.as_string::(), nullable), - DataType::Binary => Self::from_arrow(array.as_binary::(), nullable), - DataType::LargeBinary => Self::from_arrow(array.as_binary::(), nullable), - DataType::BinaryView => Self::from_arrow(array.as_binary_view(), nullable), - DataType::Utf8View => Self::from_arrow(array.as_string_view(), nullable), - DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable), - DataType::List(_) => Self::from_arrow(array.as_list::(), nullable), - DataType::LargeList(_) => Self::from_arrow(array.as_list::(), nullable), - DataType::ListView(_) => Self::from_arrow(array.as_list_view::(), nullable), - DataType::LargeListView(_) => Self::from_arrow(array.as_list_view::(), nullable), - DataType::FixedSizeList(..) => Self::from_arrow(array.as_fixed_size_list(), nullable), - DataType::Null => Self::from_arrow(as_null_array(array), nullable), + DataType::Boolean => Self::from_arrow_in(array.as_boolean(), nullable, session), + DataType::UInt8 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::UInt16 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::UInt32 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::UInt64 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Int8 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Int16 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Int32 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Int64 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Float16 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Float32 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Float64 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Utf8 => Self::from_arrow_in(array.as_string::(), nullable, session), + DataType::LargeUtf8 => Self::from_arrow_in(array.as_string::(), nullable, session), + DataType::Binary => Self::from_arrow_in(array.as_binary::(), nullable, session), + DataType::LargeBinary => { + Self::from_arrow_in(array.as_binary::(), nullable, session) + } + DataType::BinaryView => Self::from_arrow_in(array.as_binary_view(), nullable, session), + DataType::Utf8View => Self::from_arrow_in(array.as_string_view(), nullable, session), + DataType::Struct(_) => Self::from_arrow_in(array.as_struct(), nullable, session), + DataType::List(_) => Self::from_arrow_in(array.as_list::(), nullable, session), + DataType::LargeList(_) => { + Self::from_arrow_in(array.as_list::(), nullable, session) + } + DataType::ListView(_) => { + Self::from_arrow_in(array.as_list_view::(), nullable, session) + } + DataType::LargeListView(_) => { + Self::from_arrow_in(array.as_list_view::(), nullable, session) + } + DataType::FixedSizeList(..) => { + Self::from_arrow_in(array.as_fixed_size_list(), nullable, session) + } + DataType::Null => Self::from_arrow_in(as_null_array(array), nullable, session), DataType::Timestamp(u, _) => match u { - ArrowTimeUnit::Second => { - Self::from_arrow(array.as_primitive::(), nullable) - } - ArrowTimeUnit::Millisecond => { - Self::from_arrow(array.as_primitive::(), nullable) - } - ArrowTimeUnit::Microsecond => { - Self::from_arrow(array.as_primitive::(), nullable) - } - ArrowTimeUnit::Nanosecond => { - Self::from_arrow(array.as_primitive::(), nullable) - } + ArrowTimeUnit::Second => Self::from_arrow_in( + array.as_primitive::(), + nullable, + session, + ), + ArrowTimeUnit::Millisecond => Self::from_arrow_in( + array.as_primitive::(), + nullable, + session, + ), + ArrowTimeUnit::Microsecond => Self::from_arrow_in( + array.as_primitive::(), + nullable, + session, + ), + ArrowTimeUnit::Nanosecond => Self::from_arrow_in( + array.as_primitive::(), + nullable, + session, + ), }, - DataType::Date32 => Self::from_arrow(array.as_primitive::(), nullable), - DataType::Date64 => Self::from_arrow(array.as_primitive::(), nullable), + DataType::Date32 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } + DataType::Date64 => { + Self::from_arrow_in(array.as_primitive::(), nullable, session) + } DataType::Time32(u) => match u { ArrowTimeUnit::Second => { - Self::from_arrow(array.as_primitive::(), nullable) - } - ArrowTimeUnit::Millisecond => { - Self::from_arrow(array.as_primitive::(), nullable) + Self::from_arrow_in(array.as_primitive::(), nullable, session) } + ArrowTimeUnit::Millisecond => Self::from_arrow_in( + array.as_primitive::(), + nullable, + session, + ), ArrowTimeUnit::Microsecond | ArrowTimeUnit::Nanosecond => unreachable!(), }, DataType::Time64(u) => match u { - ArrowTimeUnit::Microsecond => { - Self::from_arrow(array.as_primitive::(), nullable) - } - ArrowTimeUnit::Nanosecond => { - Self::from_arrow(array.as_primitive::(), nullable) - } + ArrowTimeUnit::Microsecond => Self::from_arrow_in( + array.as_primitive::(), + nullable, + session, + ), + ArrowTimeUnit::Nanosecond => Self::from_arrow_in( + array.as_primitive::(), + nullable, + session, + ), ArrowTimeUnit::Second | ArrowTimeUnit::Millisecond => unreachable!(), }, DataType::Decimal32(..) => { - Self::from_arrow(array.as_primitive::(), nullable) + Self::from_arrow_in(array.as_primitive::(), nullable, session) } DataType::Decimal64(..) => { - Self::from_arrow(array.as_primitive::(), nullable) + Self::from_arrow_in(array.as_primitive::(), nullable, session) } DataType::Decimal128(..) => { - Self::from_arrow(array.as_primitive::(), nullable) + Self::from_arrow_in(array.as_primitive::(), nullable, session) } DataType::Decimal256(..) => { - Self::from_arrow(array.as_primitive::(), nullable) + Self::from_arrow_in(array.as_primitive::(), nullable, session) } DataType::Dictionary(key_type, _) => match key_type.as_ref() { - DataType::Int8 => Ok(DictArray::from_arrow( + DataType::Int8 => Ok(DictArray::from_arrow_in( array.as_dictionary::(), nullable, + session, )? .into_array()), - DataType::Int16 => Ok(DictArray::from_arrow( + DataType::Int16 => Ok(DictArray::from_arrow_in( array.as_dictionary::(), nullable, + session, )? .into_array()), - DataType::Int32 => Ok(DictArray::from_arrow( + DataType::Int32 => Ok(DictArray::from_arrow_in( array.as_dictionary::(), nullable, + session, )? .into_array()), - DataType::Int64 => Ok(DictArray::from_arrow( + DataType::Int64 => Ok(DictArray::from_arrow_in( array.as_dictionary::(), nullable, + session, )? .into_array()), - DataType::UInt8 => Ok(DictArray::from_arrow( + DataType::UInt8 => Ok(DictArray::from_arrow_in( array.as_dictionary::(), nullable, + session, )? .into_array()), - DataType::UInt16 => Ok(DictArray::from_arrow( + DataType::UInt16 => Ok(DictArray::from_arrow_in( array.as_dictionary::(), nullable, + session, )? .into_array()), - DataType::UInt32 => Ok(DictArray::from_arrow( + DataType::UInt32 => Ok(DictArray::from_arrow_in( array.as_dictionary::(), nullable, + session, )? .into_array()), - DataType::UInt64 => Ok(DictArray::from_arrow( + DataType::UInt64 => Ok(DictArray::from_arrow_in( array.as_dictionary::(), nullable, + session, )? .into_array()), key_dt => vortex_bail!("Unsupported dictionary key type: {key_dt}"), @@ -616,14 +726,22 @@ impl FromArrowArray<&dyn ArrowArray> for ArrayRef { } impl FromArrowArray for ArrayRef { - fn from_arrow(array: RecordBatch, nullable: bool) -> VortexResult { - ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable) + fn from_arrow_in( + array: RecordBatch, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { + ArrayRef::from_arrow_in(&arrow_array::StructArray::from(array), nullable, session) } } impl FromArrowArray<&RecordBatch> for ArrayRef { - fn from_arrow(array: &RecordBatch, nullable: bool) -> VortexResult { - Self::from_arrow(array.clone(), nullable) + fn from_arrow_in( + array: &RecordBatch, + nullable: bool, + session: &VortexSession, + ) -> VortexResult { + Self::from_arrow_in(array.clone(), nullable, session) } } @@ -683,6 +801,7 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::arrays::Decimal; use crate::arrays::FixedSizeList; use crate::arrays::List; @@ -707,10 +826,11 @@ mod tests { #[test] fn test_int8_array_conversion() { let arrow_array = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Int8Array::from(vec![1, 2, 3, 4]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -726,10 +846,11 @@ mod tests { #[test] fn test_int16_array_conversion() { let arrow_array = Int16Array::from(vec![Some(100), None, Some(300), Some(400)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Int16Array::from(vec![100, 200, 300, 400]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -745,10 +866,11 @@ mod tests { #[test] fn test_int32_array_conversion() { let arrow_array = Int32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Int32Array::from(vec![1000, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -764,10 +886,11 @@ mod tests { #[test] fn test_int64_array_conversion() { let arrow_array = Int64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Int64Array::from(vec![10000_i64, 20000, 30000, 40000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -783,10 +906,11 @@ mod tests { #[test] fn test_uint8_array_conversion() { let arrow_array = UInt8Array::from(vec![Some(1), None, Some(3), Some(4)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = UInt8Array::from(vec![1_u8, 2, 3, 4]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -802,10 +926,11 @@ mod tests { #[test] fn test_uint16_array_conversion() { let arrow_array = UInt16Array::from(vec![Some(100), None, Some(300), Some(400)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = UInt16Array::from(vec![100_u16, 200, 300, 400]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -821,10 +946,11 @@ mod tests { #[test] fn test_uint32_array_conversion() { let arrow_array = UInt32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = UInt32Array::from(vec![1000_u32, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -840,10 +966,11 @@ mod tests { #[test] fn test_uint64_array_conversion() { let arrow_array = UInt64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = UInt64Array::from(vec![10000_u64, 20000, 30000, 40000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -864,7 +991,7 @@ mod tests { Some(::Native::from_f32(3.5)), ]; let arrow_array = arrow_array::PrimitiveArray::::from(values); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let non_null_values = vec![ ::Native::from_f32(1.5), @@ -872,7 +999,8 @@ mod tests { ]; let arrow_array_non_null = arrow_array::PrimitiveArray::::from(non_null_values); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 3); assert_eq!(vortex_array_non_null.len(), 2); @@ -888,10 +1016,11 @@ mod tests { #[test] fn test_float32_array_conversion() { let arrow_array = Float32Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Float32Array::from(vec![1.5_f32, 2.5, 3.5, 4.5]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -907,10 +1036,11 @@ mod tests { #[test] fn test_float64_array_conversion() { let arrow_array = Float64Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Float64Array::from(vec![1.5_f64, 2.5, 3.5, 4.5]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -933,7 +1063,7 @@ mod tests { builder.append_value(11111); let decimal_array = builder.finish().with_precision_and_scale(10, 2).unwrap(); - let vortex_array = ArrayRef::from_arrow(&decimal_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&decimal_array, true, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); let mut builder_non_null = Decimal128Builder::with_capacity(3); @@ -945,7 +1075,8 @@ mod tests { .with_precision_and_scale(10, 2) .unwrap(); - let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&decimal_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array_non_null.len(), 3); // Verify metadata - should be DecimalArray with correct precision and scale @@ -970,7 +1101,7 @@ mod tests { builder.append_value(arrow_buffer::i256::from_i128(11111)); let decimal_array = builder.finish().with_precision_and_scale(38, 10).unwrap(); - let vortex_array = ArrayRef::from_arrow(&decimal_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&decimal_array, true, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); let mut builder_non_null = Decimal256Builder::with_capacity(3); @@ -982,7 +1113,8 @@ mod tests { .with_precision_and_scale(38, 10) .unwrap(); - let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&decimal_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array_non_null.len(), 3); // Verify metadata - should be DecimalArray with correct precision and scale @@ -1003,10 +1135,11 @@ mod tests { fn test_timestamp_second_array_conversion() { let arrow_array = TimestampSecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = TimestampSecondArray::from(vec![1000_i64, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1029,11 +1162,12 @@ mod tests { fn test_timestamp_millisecond_array_conversion() { let arrow_array = TimestampMillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = TimestampMillisecondArray::from(vec![1000_i64, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1043,11 +1177,12 @@ mod tests { fn test_timestamp_microsecond_array_conversion() { let arrow_array = TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1058,11 +1193,12 @@ mod tests { let arrow_array = TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]) .with_timezone("UTC"); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]).with_timezone("UTC"); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!( @@ -1094,10 +1230,11 @@ mod tests { fn test_timestamp_nanosecond_array_conversion() { let arrow_array = TimestampNanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = TimestampNanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1106,10 +1243,11 @@ mod tests { #[test] fn test_time32_second_array_conversion() { let arrow_array = Time32SecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Time32SecondArray::from(vec![1000_i32, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1132,10 +1270,11 @@ mod tests { fn test_time32_millisecond_array_conversion() { let arrow_array = Time32MillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Time32MillisecondArray::from(vec![1000_i32, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1145,10 +1284,11 @@ mod tests { fn test_time64_microsecond_array_conversion() { let arrow_array = Time64MicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Time64MicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1158,10 +1298,11 @@ mod tests { fn test_time64_nanosecond_array_conversion() { let arrow_array = Time64NanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Time64NanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1170,10 +1311,11 @@ mod tests { #[test] fn test_date32_array_conversion() { let arrow_array = Date32Array::from(vec![Some(18000), None, Some(18002), Some(18003)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Date32Array::from(vec![18000_i32, 18001, 18002, 18003]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1187,7 +1329,7 @@ mod tests { Some(1555286400000), Some(1555372800000), ]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = Date64Array::from(vec![ 1555200000000_i64, @@ -1195,7 +1337,8 @@ mod tests { 1555286400000, 1555372800000, ]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1205,10 +1348,11 @@ mod tests { #[test] fn test_utf8_array_conversion() { let arrow_array = StringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = StringArray::from(vec!["hello", "world", "test", "vortex"]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1225,10 +1369,11 @@ mod tests { fn test_large_utf8_array_conversion() { let arrow_array = LargeStringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = LargeStringArray::from(vec!["hello", "world", "test", "vortex"]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1242,7 +1387,7 @@ mod tests { Some("world".as_bytes()), Some("test".as_bytes()), ]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = BinaryArray::from(vec![ "hello".as_bytes(), @@ -1250,7 +1395,8 @@ mod tests { "test".as_bytes(), "vortex".as_bytes(), ]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1264,7 +1410,7 @@ mod tests { Some("world".as_bytes()), Some("test".as_bytes()), ]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = LargeBinaryArray::from(vec![ "hello".as_bytes(), @@ -1272,7 +1418,8 @@ mod tests { "test".as_bytes(), "vortex".as_bytes(), ]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1286,7 +1433,7 @@ mod tests { builder.append_value("world"); builder.append_value("test"); let arrow_array = builder.finish(); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let mut builder_non_null = StringViewBuilder::new(); builder_non_null.append_value("hello"); @@ -1294,7 +1441,8 @@ mod tests { builder_non_null.append_value("test"); builder_non_null.append_value("vortex"); let arrow_array_non_null = builder_non_null.finish(); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1326,7 +1474,7 @@ mod tests { builder.append_value(b"world"); builder.append_value(b"test"); let arrow_array = builder.finish(); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let mut builder_non_null = BinaryViewBuilder::new(); builder_non_null.append_value(b"hello"); @@ -1334,7 +1482,8 @@ mod tests { builder_non_null.append_value(b"test"); builder_non_null.append_value(b"vortex"); let arrow_array_non_null = builder_non_null.finish(); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1362,10 +1511,11 @@ mod tests { #[test] fn test_boolean_array_conversion() { let arrow_array = BooleanArray::from(vec![Some(true), None, Some(false), Some(true)]); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); let arrow_array_non_null = BooleanArray::from(vec![true, false, true, false]); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); assert_eq!(vortex_array_non_null.len(), 4); @@ -1389,7 +1539,7 @@ mod tests { None, ); - let vortex_array = ArrayRef::from_arrow(&arrow_array, false).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 3); // Verify metadata - should be StructArray with correct field names @@ -1410,7 +1560,8 @@ mod tests { ]))), ); - let vortex_nullable_array = ArrayRef::from_arrow(&nullable_array, true).unwrap(); + let vortex_nullable_array = + ArrayRef::from_arrow_in(&nullable_array, true, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_nullable_array.len(), 3); // Verify metadata for nullable struct @@ -1429,7 +1580,7 @@ mod tests { builder.append_value([Some(4), Some(5)]); let arrow_array = builder.finish(); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 3); // Verify metadata - should be ListArray with correct offsets @@ -1444,7 +1595,8 @@ mod tests { builder_non_null.append_value([Some(4), Some(5)]); let arrow_array_non_null = builder_non_null.finish(); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array_non_null.len(), 2); // Verify metadata for non-nullable list @@ -1462,7 +1614,7 @@ mod tests { builder.append_value([Some(4), Some(5)]); let arrow_array = builder.finish(); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 3); // Verify metadata - should be ListArray with correct offsets (I64 for large lists) @@ -1477,7 +1629,8 @@ mod tests { builder_non_null.append_value([Some(4), Some(5)]); let arrow_array_non_null = builder_non_null.finish(); - let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false).unwrap(); + let vortex_array_non_null = + ArrayRef::from_arrow_in(&arrow_array_non_null, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array_non_null.len(), 2); // Verify metadata for non-nullable large list @@ -1510,7 +1663,7 @@ mod tests { let arrow_array = ArrowFixedSizeListArray::try_new(Arc::clone(&field), 3, Arc::new(values), None) .unwrap(); - let vortex_array = ArrayRef::from_arrow(&arrow_array, false).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); @@ -1543,7 +1696,8 @@ mod tests { Some(null_buffer), ) .unwrap(); - let vortex_array_nullable = ArrayRef::from_arrow(&arrow_array_nullable, true).unwrap(); + let vortex_array_nullable = + ArrayRef::from_arrow_in(&arrow_array_nullable, true, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array_nullable.len(), 3); @@ -1583,7 +1737,7 @@ mod tests { ) .unwrap(); - let vortex_array = ArrayRef::from_arrow(&arrow_array, false).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); // Verify metadata - should be ListViewArray with correct offsets and sizes @@ -1609,7 +1763,8 @@ mod tests { ) .unwrap(); - let vortex_array_nullable = ArrayRef::from_arrow(&arrow_array_nullable, true).unwrap(); + let vortex_array_nullable = + ArrayRef::from_arrow_in(&arrow_array_nullable, true, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array_nullable.len(), 4); // Test LargeListView (i64 offsets and sizes) @@ -1625,7 +1780,8 @@ mod tests { ) .unwrap(); - let large_vortex_array = ArrayRef::from_arrow(&large_arrow_array, false).unwrap(); + let large_vortex_array = + ArrayRef::from_arrow_in(&large_arrow_array, false, &LEGACY_SESSION).unwrap(); assert_eq!(large_vortex_array.len(), 4); // Verify metadata for large ListView @@ -1643,7 +1799,7 @@ mod tests { #[test] fn test_null_array_conversion() { let arrow_array = NullArray::new(5); - let vortex_array = ArrayRef::from_arrow(&arrow_array, true).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&arrow_array, true, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 5); } @@ -1693,7 +1849,7 @@ mod tests { let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap(); - let vortex_array = ArrayRef::from_arrow(record_batch, false).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(record_batch, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); // Test with reference @@ -1707,7 +1863,7 @@ mod tests { let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap(); - let vortex_array = ArrayRef::from_arrow(&record_batch, false).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(&record_batch, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); } @@ -1716,17 +1872,17 @@ mod tests { fn test_dyn_array_conversion() { let int_array = Int32Array::from(vec![1, 2, 3, 4]); let dyn_array: &dyn ArrowArray = &int_array; - let vortex_array = ArrayRef::from_arrow(dyn_array, false).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(dyn_array, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 4); let string_array = StringArray::from(vec!["a", "b", "c"]); let dyn_array: &dyn ArrowArray = &string_array; - let vortex_array = ArrayRef::from_arrow(dyn_array, false).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(dyn_array, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 3); let bool_array = BooleanArray::from(vec![true, false, true]); let dyn_array: &dyn ArrowArray = &bool_array; - let vortex_array = ArrayRef::from_arrow(dyn_array, false).unwrap(); + let vortex_array = ArrayRef::from_arrow_in(dyn_array, false, &LEGACY_SESSION).unwrap(); assert_eq!(vortex_array.len(), 3); } @@ -1741,7 +1897,12 @@ mod tests { )])), 1, ); - ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true).unwrap(); + ArrayRef::from_arrow_in( + null_struct_array_with_non_nullable_field.as_ref(), + true, + &LEGACY_SESSION, + ) + .unwrap(); } #[test] @@ -1758,7 +1919,12 @@ mod tests { )])), 1, ); - ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true).unwrap(); + ArrayRef::from_arrow_in( + null_struct_array_with_non_nullable_field.as_ref(), + true, + &LEGACY_SESSION, + ) + .unwrap(); } #[test] @@ -1773,6 +1939,11 @@ mod tests { 1, ); - ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true).unwrap(); + ArrayRef::from_arrow_in( + null_struct_array_with_non_nullable_field.as_ref(), + true, + &LEGACY_SESSION, + ) + .unwrap(); } } diff --git a/vortex-array/src/arrow/datum.rs b/vortex-array/src/arrow/datum.rs index 66c457d3727..5c2da1d051d 100644 --- a/vortex-array/src/arrow/datum.rs +++ b/vortex-array/src/arrow/datum.rs @@ -93,7 +93,7 @@ pub fn from_arrow_array_with_len(array: A, len: usize, nullable: bool) -> Vor where ArrayRef: FromArrowArray, { - let array = ArrayRef::from_arrow(array, nullable)?; + let array = ArrayRef::from_arrow_in(array, nullable, &LEGACY_SESSION)?; if array.len() == len { return Ok(array); } diff --git a/vortex-array/src/arrow/executor/byte_view.rs b/vortex-array/src/arrow/executor/byte_view.rs index b88b1895d53..ac04885264e 100644 --- a/vortex-array/src/arrow/executor/byte_view.rs +++ b/vortex-array/src/arrow/executor/byte_view.rs @@ -71,7 +71,10 @@ pub(super) fn to_arrow_byte_view( // We do this in case the vortex array is Utf8, and we want Binary or vice versa. By casting // first, we may push this down through the Vortex array tree. We choose nullable to be most // flexible since there's no prescribed nullability in Arrow types. - let array = array.cast(DType::from_arrow((&T::DATA_TYPE, Nullability::Nullable)))?; + let array = array.cast(DType::from_arrow_in( + (&T::DATA_TYPE, Nullability::Nullable), + ctx.session(), + ))?; let varbinview = array.execute::(ctx)?; canonical_varbinview_to_arrow::(&varbinview, ctx) diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index 909fd78059d..b58305dac4d 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -86,7 +86,7 @@ pub(super) fn to_arrow_struct( // Otherwise, we fall back to executing to a StructArray. let array = if let Some(fields) = target_fields { - let vx_fields = StructFields::from_arrow(fields); + let vx_fields = StructFields::from_arrow_in(fields, ctx.session()); // We apply a cast to ensure we push down casting where possible into the struct fields. array.cast(DType::Struct( vx_fields, @@ -351,7 +351,8 @@ mod tests { )?; let orig_dtype = array.dtype().clone(); let arrow_array = array.into_array().execute_arrow(None, &mut ctx)?; - let from_arrow = array::ArrayRef::from_arrow(arrow_array.as_ref(), false)?; + let from_arrow = + array::ArrayRef::from_arrow_in(arrow_array.as_ref(), false, ctx.session())?; assert_eq!(&orig_dtype, from_arrow.dtype()); Ok(()) } diff --git a/vortex-array/src/arrow/iter.rs b/vortex-array/src/arrow/iter.rs index f05bb0a4e7b..4afd00b6883 100644 --- a/vortex-array/src/arrow/iter.rs +++ b/vortex-array/src/arrow/iter.rs @@ -6,6 +6,7 @@ use vortex_error::VortexError; use vortex_error::VortexResult; use crate::ArrayRef; +use crate::LEGACY_SESSION; use crate::arrow::FromArrowArray; use crate::dtype::DType; use crate::dtype::arrow::FromArrowType; @@ -36,8 +37,11 @@ impl Iterator for ArrowArrayStreamAdapter { let batch = self.stream.next()?; Some(batch.map_err(VortexError::from).and_then(|b| { - debug_assert_eq!(&self.dtype, &DType::from_arrow(b.schema())); - ArrayRef::from_arrow(b, false) + debug_assert_eq!( + &self.dtype, + &DType::from_arrow_in(b.schema(), &LEGACY_SESSION) + ); + ArrayRef::from_arrow_in(b, false, &LEGACY_SESSION) })) } } diff --git a/vortex-array/src/arrow/mod.rs b/vortex-array/src/arrow/mod.rs index efc83aa6af6..4d9c17c91af 100644 --- a/vortex-array/src/arrow/mod.rs +++ b/vortex-array/src/arrow/mod.rs @@ -6,6 +6,7 @@ use arrow_array::ArrayRef as ArrowArrayRef; use arrow_schema::DataType; use vortex_error::VortexResult; +use vortex_session::VortexSession; mod convert; mod datum; @@ -24,10 +25,15 @@ use crate::ArrayRef; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; -pub trait FromArrowArray { - fn from_arrow(array: A, nullable: bool) -> VortexResult - where - Self: Sized; +pub trait FromArrowArray: Sized { + /// Convert an Arrow array to a Vortex array, looking up extension types in `session`. + fn from_arrow_in(array: A, nullable: bool, session: &VortexSession) -> VortexResult; + + /// Convert an Arrow array to a Vortex array using the legacy global session. + #[deprecated(note = "Use `from_arrow_in` instead")] + fn from_arrow(array: A, nullable: bool) -> VortexResult { + Self::from_arrow_in(array, nullable, &LEGACY_SESSION) + } } #[deprecated(note = "Use `execute_arrow(None, ctx)` or `execute_arrow(Some(dt), ctx)` instead")] diff --git a/vortex-array/src/arrow/record_batch.rs b/vortex-array/src/arrow/record_batch.rs index b57c307aed0..0f7902ec6c4 100644 --- a/vortex-array/src/arrow/record_batch.rs +++ b/vortex-array/src/arrow/record_batch.rs @@ -6,6 +6,7 @@ use arrow_array::cast::AsArray; use arrow_schema::DataType; use arrow_schema::Schema; use vortex_error::VortexResult; +use vortex_session::VortexSession; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; @@ -14,16 +15,27 @@ use crate::arrays::StructArray; use crate::arrow::ArrowArrayExecutor; impl StructArray { - pub fn into_record_batch_with_schema( + /// Convert a [`StructArray`] to a [`RecordBatch`] with the given schema, using `session`. + pub fn into_record_batch_with_schema_in( self, schema: impl AsRef, + session: &VortexSession, ) -> VortexResult { let data_type = DataType::Struct(schema.as_ref().fields.clone()); let array_ref = self .into_array() - .execute_arrow(Some(&data_type), &mut LEGACY_SESSION.create_execution_ctx())?; + .execute_arrow(Some(&data_type), &mut session.create_execution_ctx())?; Ok(RecordBatch::from(array_ref.as_struct())) } + + /// Convert a [`StructArray`] to a [`RecordBatch`] using the legacy global session. + #[deprecated(note = "Use `into_record_batch_with_schema_in` instead")] + pub fn into_record_batch_with_schema( + self, + schema: impl AsRef, + ) -> VortexResult { + self.into_record_batch_with_schema_in(schema, &LEGACY_SESSION) + } } #[cfg(test)] @@ -35,6 +47,7 @@ mod tests { use arrow_schema::FieldRef; use arrow_schema::Schema; + use crate::LEGACY_SESSION; use crate::arrow::record_batch::StructArray; use crate::builders::ArrayBuilder; use crate::builders::ListBuilder; @@ -69,7 +82,9 @@ mod tests { DataType::LargeListView(FieldRef::new(Field::new_list_field(DataType::Int32, false))), true, )])); - let rb = array.into_record_batch_with_schema(arrow_schema).unwrap(); + let rb = array + .into_record_batch_with_schema_in(arrow_schema, &LEGACY_SESSION) + .unwrap(); let xs = rb.column(0); assert_eq!( diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index 150172b0df8..e9119ff4218 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -1165,7 +1165,7 @@ mod test { nulls.finish(), ); - let vortex_struct = ArrayRef::from_arrow(&arrow_struct, true).unwrap(); + let vortex_struct = ArrayRef::from_arrow_in(&arrow_struct, true, ctx.session()).unwrap(); assert_eq!( &arrow_struct, @@ -1193,7 +1193,7 @@ mod test { ); let list_data_type = arrow_list.data_type(); - let vortex_list = ArrayRef::from_arrow(&arrow_list, true).unwrap(); + let vortex_list = ArrayRef::from_arrow_in(&arrow_list, true, ctx.session()).unwrap(); let rt_arrow_list = vortex_list .execute_arrow(Some(list_data_type), &mut ctx) diff --git a/vortex-array/src/dtype/arrow.rs b/vortex-array/src/dtype/arrow.rs index 17af749cfc0..c954e56dc07 100644 --- a/vortex-array/src/dtype/arrow.rs +++ b/vortex-array/src/dtype/arrow.rs @@ -29,7 +29,9 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use crate::LEGACY_SESSION; use crate::dtype::DType; use crate::dtype::DecimalDType; use crate::dtype::FieldName; @@ -45,8 +47,14 @@ use crate::extension::datetime::Timestamp; /// Trait for converting Arrow types to Vortex types. pub trait FromArrowType: Sized { - /// Convert the Arrow type to a Vortex type. - fn from_arrow(value: T) -> Self; + /// Convert the Arrow type to a Vortex type, looking up extension types in `session`. + fn from_arrow_in(value: T, session: &VortexSession) -> Self; + + /// Convert the Arrow type to a Vortex type using the legacy global session. + #[deprecated(note = "Use `from_arrow_in` instead")] + fn from_arrow(value: T) -> Self { + Self::from_arrow_in(value, &LEGACY_SESSION) + } } /// Trait for converting Vortex types to Arrow types. @@ -125,33 +133,36 @@ impl TryFrom for ArrowTimeUnit { } impl FromArrowType for DType { - fn from_arrow(value: SchemaRef) -> Self { - Self::from_arrow(value.as_ref()) + fn from_arrow_in(value: SchemaRef, session: &VortexSession) -> Self { + Self::from_arrow_in(value.as_ref(), session) } } impl FromArrowType<&Schema> for DType { - fn from_arrow(value: &Schema) -> Self { + fn from_arrow_in(value: &Schema, session: &VortexSession) -> Self { Self::Struct( - StructFields::from_arrow(value.fields()), + StructFields::from_arrow_in(value.fields(), session), Nullability::NonNullable, // Must match From for Array ) } } impl FromArrowType<&Fields> for StructFields { - fn from_arrow(value: &Fields) -> Self { + fn from_arrow_in(value: &Fields, session: &VortexSession) -> Self { StructFields::from_iter(value.into_iter().map(|f| { ( FieldName::from(f.name().as_str()), - DType::from_arrow(f.as_ref()), + DType::from_arrow_in(f.as_ref(), session), ) })) } } impl FromArrowType<(&DataType, Nullability)> for DType { - fn from_arrow((data_type, nullability): (&DataType, Nullability)) -> Self { + fn from_arrow_in( + (data_type, nullability): (&DataType, Nullability), + session: &VortexSession, + ) -> Self { if data_type.is_integer() || data_type.is_floating() { return DType::Primitive( PType::try_from_arrow(data_type).vortex_expect("arrow float/integer to ptype"), @@ -188,20 +199,23 @@ impl FromArrowType<(&DataType, Nullability)> for DType { DataType::List(e) | DataType::LargeList(e) | DataType::ListView(e) - | DataType::LargeListView(e) => { - DType::List(Arc::new(Self::from_arrow(e.as_ref())), nullability) - } + | DataType::LargeListView(e) => DType::List( + Arc::new(Self::from_arrow_in(e.as_ref(), session)), + nullability, + ), DataType::FixedSizeList(e, size) => DType::FixedSizeList( - Arc::new(Self::from_arrow(e.as_ref())), + Arc::new(Self::from_arrow_in(e.as_ref(), session)), *size as u32, nullability, ), - DataType::Struct(f) => DType::Struct(StructFields::from_arrow(f), nullability), + DataType::Struct(f) => { + DType::Struct(StructFields::from_arrow_in(f, session), nullability) + } DataType::Dictionary(_, value_type) => { - Self::from_arrow((value_type.as_ref(), nullability)) + Self::from_arrow_in((value_type.as_ref(), nullability), session) } DataType::RunEndEncoded(_, value_type) => { - Self::from_arrow((value_type.data_type(), nullability)) + Self::from_arrow_in((value_type.data_type(), nullability), session) } _ => unimplemented!("Arrow data type not yet supported: {:?}", data_type), } @@ -209,7 +223,7 @@ impl FromArrowType<(&DataType, Nullability)> for DType { } impl FromArrowType<&Field> for DType { - fn from_arrow(field: &Field) -> Self { + fn from_arrow_in(field: &Field, session: &VortexSession) -> Self { if field .metadata() .get("ARROW:extension:name") @@ -218,7 +232,7 @@ impl FromArrowType<&Field> for DType { { return DType::Variant(field.is_nullable().into()); } - Self::from_arrow((field.data_type(), field.is_nullable().into())) + Self::from_arrow_in((field.data_type(), field.is_nullable().into()), session) } } @@ -373,6 +387,7 @@ mod test { use rstest::rstest; use super::*; + use crate::LEGACY_SESSION; use crate::dtype::DType; use crate::dtype::FieldName; use crate::dtype::FieldNames; @@ -536,7 +551,8 @@ mod test { ); let arrow_dtype = original_dtype.to_arrow_dtype().unwrap(); - let roundtripped_dtype = DType::from_arrow((&arrow_dtype, Nullability::NonNullable)); + let roundtripped_dtype = + DType::from_arrow_in((&arrow_dtype, Nullability::NonNullable), &LEGACY_SESSION); assert_eq!(original_dtype, roundtripped_dtype); } @@ -557,7 +573,8 @@ mod test { DType::struct_([("\u{7}=outer", inner_struct)], Nullability::NonNullable); let arrow_dtype = original_dtype.to_arrow_dtype().unwrap(); - let roundtripped_dtype = DType::from_arrow((&arrow_dtype, Nullability::NonNullable)); + let roundtripped_dtype = + DType::from_arrow_in((&arrow_dtype, Nullability::NonNullable), &LEGACY_SESSION); assert_eq!(original_dtype, roundtripped_dtype); } diff --git a/vortex-array/src/scalar_fn/fns/binary/boolean.rs b/vortex-array/src/scalar_fn/fns/binary/boolean.rs index 68ef0d61156..855581e3a86 100644 --- a/vortex-array/src/scalar_fn/fns/binary/boolean.rs +++ b/vortex-array/src/scalar_fn/fns/binary/boolean.rs @@ -70,7 +70,7 @@ fn arrow_execute_boolean( other => return Err(vortex_err!("Not a boolean operator: {other}")), }; - ArrayRef::from_arrow(&array, nullable) + ArrayRef::from_arrow_in(&array, nullable, ctx.session()) } /// Constant-folds a boolean operation between two constant arrays. diff --git a/vortex-bench/src/conversions.rs b/vortex-bench/src/conversions.rs index 3f21ab30ba0..d863c7ae8c5 100644 --- a/vortex-bench/src/conversions.rs +++ b/vortex-bench/src/conversions.rs @@ -20,6 +20,7 @@ use tracing::trace; use vortex::VortexSessionDefault; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::VortexSessionExecute; use vortex::array::arrays::ChunkedArray; use vortex::array::arrow::FromArrowArray; @@ -91,7 +92,7 @@ pub fn parquet_to_vortex_stream( ) -> impl futures::Stream> { reader.map(move |result| { result.map_err(|e| vortex_err!(External: e)).and_then(|rb| { - let chunk = ArrayRef::from_arrow(rb, false)?; + let chunk = ArrayRef::from_arrow_in(rb, false, &LEGACY_SESSION)?; let mut builder = builder_with_capacity(chunk.dtype(), chunk.len()); // Canonicalize the chunk. @@ -115,7 +116,7 @@ pub async fn convert_parquet_file_to_vortex( ) -> anyhow::Result<()> { let file = File::open(parquet_path).await?; let builder = ParquetRecordBatchStreamBuilder::new(file).await?; - let dtype = DType::from_arrow(builder.schema().as_ref()); + let dtype = DType::from_arrow_in(builder.schema().as_ref(), &LEGACY_SESSION); let stream = parquet_to_vortex_stream(builder.build()?); diff --git a/vortex-bench/src/tpch/tpchgen.rs b/vortex-bench/src/tpch/tpchgen.rs index 5dbeb380a7e..b395fa3d61c 100644 --- a/vortex-bench/src/tpch/tpchgen.rs +++ b/vortex-bench/src/tpch/tpchgen.rs @@ -32,6 +32,7 @@ use tpchgen::generators::SupplierGenerator; use tpchgen_arrow::RecordBatchIterator; use tracing::info; use vortex::array::ArrayRef; +use vortex::array::LEGACY_SESSION; use vortex::array::arrow::FromArrowArray; use vortex::array::stream::ArrayStreamAdapter; use vortex::dtype::DType; @@ -403,7 +404,7 @@ impl VortexWriter { ) -> Result { // Increase buffer size to avoid backpressure issues let (sender, receiver) = mpsc::channel(2); - let dtype = DType::from_arrow(schema); + let dtype = DType::from_arrow_in(schema, &LEGACY_SESSION); let file_path = path; let write_task = Some(tokio::spawn(async move { let stream = ArrayStreamAdapter::new(dtype, ReceiverStream::new(receiver)); @@ -428,7 +429,7 @@ impl VortexWriter { #[async_trait::async_trait] impl FileWriter for VortexWriter { async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { - let array = ArrayRef::from_arrow(batch, false)?; + let array = ArrayRef::from_arrow_in(batch, false, &LEGACY_SESSION)?; self.sender .as_ref() .vortex_expect("sender closed early") diff --git a/vortex-cxx/src/dtype.rs b/vortex-cxx/src/dtype.rs index 0c125bcf822..208fb091837 100644 --- a/vortex-cxx/src/dtype.rs +++ b/vortex-cxx/src/dtype.rs @@ -7,6 +7,7 @@ use std::fmt::Formatter; use anyhow::Result; use arrow_array::ffi::FFI_ArrowSchema; use arrow_schema::Field; +use vortex::array::LEGACY_SESSION; use vortex::dtype::DType as RustDType; use vortex::dtype::DecimalDType; use vortex::dtype::Nullability; @@ -89,6 +90,9 @@ pub(crate) unsafe fn from_arrow(ffi_schema: *mut u8, non_nullable: bool) -> Resu let arrow_schema = unsafe { FFI_ArrowSchema::from_raw(ffi_schema as *mut FFI_ArrowSchema) }; let arrow_dtype = arrow_schema::DataType::try_from(&arrow_schema)?; Ok(Box::new(DType { - inner: RustDType::from_arrow(&Field::new("_", arrow_dtype, !non_nullable)), + inner: RustDType::from_arrow_in( + &Field::new("_", arrow_dtype, !non_nullable), + &LEGACY_SESSION, + ), })) } diff --git a/vortex-cxx/src/write.rs b/vortex-cxx/src/write.rs index ff77c5a3dc5..46bcc7e67b2 100644 --- a/vortex-cxx/src/write.rs +++ b/vortex-cxx/src/write.rs @@ -6,6 +6,7 @@ use arrow_array::RecordBatchReader; use arrow_array::ffi_stream::ArrowArrayStreamReader; use arrow_array::ffi_stream::FFI_ArrowArrayStream; use vortex::array::ArrayRef; +use vortex::array::LEGACY_SESSION; use vortex::array::arrow::FromArrowArray; use vortex::array::iter::ArrayIteratorAdapter; use vortex::array::iter::ArrayIteratorExt; @@ -34,11 +35,11 @@ pub(crate) fn write_options_new() -> Box { /// Convert an ArrowArrayStreamReader to a Vortex ArrayStream fn arrow_stream_to_vortex_stream(reader: ArrowArrayStreamReader) -> Result { let array_iter = ArrayIteratorAdapter::new( - DType::from_arrow(reader.schema()), + DType::from_arrow_in(reader.schema(), &LEGACY_SESSION), reader.map(|result| { - result - .map_err(VortexError::from) - .and_then(|record_batch| ArrayRef::from_arrow(record_batch, false)) + result.map_err(VortexError::from).and_then(|record_batch| { + ArrayRef::from_arrow_in(record_batch, false, &LEGACY_SESSION) + }) }), ); diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index 6a9ac0fdc52..fc6497490b9 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -19,6 +19,7 @@ use datafusion_physical_expr::utils::collect_columns; use datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr; use datafusion_physical_plan::expressions as df_expr; use itertools::Itertools; +use vortex::array::LEGACY_SESSION; use vortex::dtype::DType; use vortex::dtype::Nullability; use vortex::dtype::arrow::FromArrowType; @@ -225,7 +226,10 @@ impl ExpressionConvertor for DefaultExpressionConvertor { } if let Some(cast_expr) = df.as_any().downcast_ref::() { - let cast_dtype = DType::from_arrow((cast_expr.cast_type(), Nullability::Nullable)); + let cast_dtype = DType::from_arrow_in( + (cast_expr.cast_type(), Nullability::Nullable), + &LEGACY_SESSION, + ); let child = self.convert(cast_expr.expr().as_ref())?; return Ok(cast(child, cast_dtype)); } @@ -233,7 +237,10 @@ impl ExpressionConvertor for DefaultExpressionConvertor { if let Some(cast_col_expr) = df.as_any().downcast_ref::() { let target = cast_col_expr.target_field(); - let target_dtype = DType::from_arrow((target.data_type(), target.is_nullable().into())); + let target_dtype = DType::from_arrow_in( + (target.data_type(), target.is_nullable().into()), + &LEGACY_SESSION, + ); let child = self.convert(cast_col_expr.expr().as_ref())?; return Ok(cast(child, target_dtype)); } @@ -973,7 +980,8 @@ mod tests { let vortex_expr = expr_convertor.try_convert_case_expr(&case_expr).unwrap(); // Convert batch to Vortex array - let vortex_array: ArrayRef = ArrayRef::from_arrow(&batch, false).unwrap(); + let vortex_array: ArrayRef = + ArrayRef::from_arrow_in(&batch, false, &LEGACY_SESSION).unwrap(); // Apply Vortex expression let session = VortexSession::default(); diff --git a/vortex-datafusion/src/convert/scalars.rs b/vortex-datafusion/src/convert/scalars.rs index a9b9649fd59..380bee6c46f 100644 --- a/vortex-datafusion/src/convert/scalars.rs +++ b/vortex-datafusion/src/convert/scalars.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use datafusion_common::ScalarValue; +use vortex::array::LEGACY_SESSION; use vortex::buffer::ByteBuffer; use vortex::dtype::DType; use vortex::dtype::DecimalDType; @@ -219,7 +220,10 @@ impl FromDataFusion for Scalar { ScalarValue::Date32(v) | ScalarValue::Time32Second(v) | ScalarValue::Time32Millisecond(v) => { - let dtype = DType::from_arrow((&value.data_type(), Nullability::Nullable)); + let dtype = DType::from_arrow_in( + (&value.data_type(), Nullability::Nullable), + &LEGACY_SESSION, + ); Scalar::try_new(dtype, v.map(vortex::scalar::ScalarValue::from)) .vortex_expect("unable to create a time `Scalar`") } @@ -230,7 +234,10 @@ impl FromDataFusion for Scalar { | ScalarValue::TimestampMillisecond(v, _) | ScalarValue::TimestampMicrosecond(v, _) | ScalarValue::TimestampNanosecond(v, _) => { - let dtype = DType::from_arrow((&value.data_type(), Nullability::Nullable)); + let dtype = DType::from_arrow_in( + (&value.data_type(), Nullability::Nullable), + &LEGACY_SESSION, + ); Scalar::try_new(dtype, v.map(vortex::scalar::ScalarValue::from)) .vortex_expect("unable to create a time `Scalar`") } diff --git a/vortex-datafusion/src/lib.rs b/vortex-datafusion/src/lib.rs index e127e3e33ee..c98627cebe6 100644 --- a/vortex-datafusion/src/lib.rs +++ b/vortex-datafusion/src/lib.rs @@ -153,6 +153,7 @@ mod common_tests { use url::Url; use vortex::VortexSessionDefault; use vortex::array::ArrayRef; + use vortex::array::LEGACY_SESSION; use vortex::array::arrow::FromArrowArray; use vortex::file::WriteOptionsSessionExt; use vortex::io::VortexWrite; @@ -210,7 +211,7 @@ mod common_tests { where P: Into, { - let array = ArrayRef::from_arrow(batch, false)?; + let array = ArrayRef::from_arrow_in(batch, false, &LEGACY_SESSION)?; let mut write = ObjectStoreWrite::new(Arc::clone(&self.store), &path.into()).await?; VX_SESSION .write_options() diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index cb24be972c2..7b84d34f7cb 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -44,6 +44,7 @@ use futures::stream; use object_store::ObjectMeta; use object_store::ObjectStore; use vortex::VortexSessionDefault; +use vortex::array::LEGACY_SESSION; use vortex::array::memory::MemorySessionExt; use vortex::dtype::DType; use vortex::dtype::Nullability; @@ -526,7 +527,7 @@ impl FileFormat for VortexFormat { let column_size = stats_set.get_as::(Stat::UncompressedSizeInBytes, &PType::U64.into()); - let target_dtype = DType::from_arrow(field.as_ref()); + let target_dtype = DType::from_arrow_in(field.as_ref(), &LEGACY_SESSION); let min = scalar_stat_to_df( Stat::Min, stats_set.get(Stat::Min), diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index acff2a22806..9d4aba57252 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -568,6 +568,7 @@ mod tests { use rstest::rstest; use vortex::VortexSessionDefault; use vortex::array::ArrayRef; + use vortex::array::LEGACY_SESSION; use vortex::array::arrow::FromArrowArray; use vortex::buffer::Buffer; use vortex::file::WriteOptionsSessionExt; @@ -645,7 +646,7 @@ mod tests { path: &str, rb: RecordBatch, ) -> anyhow::Result { - let array = ArrayRef::from_arrow(rb, false)?; + let array = ArrayRef::from_arrow_in(rb, false, &LEGACY_SESSION)?; let path = Path::parse(path)?; let mut write = ObjectStoreWrite::new(object_store, &path).await?; diff --git a/vortex-datafusion/src/persistent/sink.rs b/vortex-datafusion/src/persistent/sink.rs index 263579a8cd5..7250bb3e5ee 100644 --- a/vortex-datafusion/src/persistent/sink.rs +++ b/vortex-datafusion/src/persistent/sink.rs @@ -26,6 +26,7 @@ use object_store::ObjectStore; use object_store::path::Path; use tokio_stream::wrappers::ReceiverStream; use vortex::array::ArrayRef; +use vortex::array::LEGACY_SESSION; use vortex::array::arrow::FromArrowArray; use vortex::array::stream::ArrayStreamAdapter; use vortex::dtype::DType; @@ -115,12 +116,13 @@ impl FileSink for VortexSink { let session = self.session.clone(); let object_store = Arc::clone(&object_store); let writer_schema = get_writer_schema(&self.config); - let dtype = DType::from_arrow(writer_schema); + let dtype = DType::from_arrow_in(writer_schema, &LEGACY_SESSION); // We need to spawn work because there's a dependency between the different files. If one file has too many batches buffered, // the demux task might deadlock itself. file_write_tasks.spawn(async move { - let stream = ReceiverStream::new(rx).map(move |rb| ArrayRef::from_arrow(rb, false)); + let stream = ReceiverStream::new(rx) + .map(move |rb| ArrayRef::from_arrow_in(rb, false, &LEGACY_SESSION)); let stream_adapter = ArrayStreamAdapter::new(dtype, stream); diff --git a/vortex-jni/src/dtype.rs b/vortex-jni/src/dtype.rs index f748135ade1..e401dfb030d 100644 --- a/vortex-jni/src/dtype.rs +++ b/vortex-jni/src/dtype.rs @@ -10,6 +10,7 @@ use arrow_array::ffi::FFI_ArrowSchema; use arrow_schema::DataType; use arrow_schema::FieldRef; use arrow_schema::Fields; +use vortex::array::LEGACY_SESSION; use vortex::dtype::DType; use vortex::dtype::arrow::FromArrowType; use vortex::error::VortexResult; @@ -74,5 +75,5 @@ pub(crate) fn strip_views(data_type: DataType) -> DataType { pub(crate) fn import_dtype_from_arrow(schema_addr: i64) -> VortexResult { let ffi_schema = unsafe { &*(schema_addr as *const FFI_ArrowSchema) }; let arrow_schema = arrow_schema::Schema::try_from(ffi_schema)?; - Ok(DType::from_arrow(&arrow_schema)) + Ok(DType::from_arrow_in(&arrow_schema, &LEGACY_SESSION)) } diff --git a/vortex-jni/src/writer.rs b/vortex-jni/src/writer.rs index e8ee86434c1..26ccdf66f2d 100644 --- a/vortex-jni/src/writer.rs +++ b/vortex-jni/src/writer.rs @@ -25,6 +25,7 @@ use jni::sys::jlong; use object_store::path::Path; use url::Url; use vortex::array::ArrayRef; +use vortex::array::LEGACY_SESSION; use vortex::array::arrow::FromArrowArray; use vortex::array::stream::ArrayStreamAdapter; use vortex::dtype::DType; @@ -88,7 +89,7 @@ impl NativeWriter { } fn write_record_batch(&self, batch: RecordBatch) -> VortexResult<()> { - let vortex_batch = ArrayRef::from_arrow(batch, false)?; + let vortex_batch = ArrayRef::from_arrow_in(batch, false, &LEGACY_SESSION)?; if !vortex_batch.dtype().eq(&self.write_schema) { return Err(vortex_err!( "write schema mismatch: expected {}, got {}", diff --git a/vortex-layout/src/scan/arrow.rs b/vortex-layout/src/scan/arrow.rs index 875db04e4df..15dcadbab23 100644 --- a/vortex-layout/src/scan/arrow.rs +++ b/vortex-layout/src/scan/arrow.rs @@ -126,6 +126,7 @@ mod tests { use arrow_schema::Field; use arrow_schema::Schema; use vortex_array::ArrayRef; + use vortex_array::LEGACY_SESSION; use vortex_array::arrow::FromArrowArray; use vortex_error::VortexResult; @@ -153,7 +154,7 @@ mod tests { ); // Convert to Vortex - ArrayRef::from_arrow(&struct_array, true) + ArrayRef::from_arrow_in(&struct_array, true, &LEGACY_SESSION) } fn create_arrow_schema() -> Arc { diff --git a/vortex-python/src/arrays/from_arrow.rs b/vortex-python/src/arrays/from_arrow.rs index beb142a1625..729d3538f76 100644 --- a/vortex-python/src/arrays/from_arrow.rs +++ b/vortex-python/src/arrays/from_arrow.rs @@ -12,6 +12,7 @@ use pyo3::intern; use pyo3::prelude::*; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::arrays::ChunkedArray; use vortex::array::arrow::FromArrowArray; use vortex::dtype::DType; @@ -37,7 +38,8 @@ pub(super) fn from_arrow(obj: &Borrowed<'_, '_, PyAny>) -> PyVortexResult> = obj.getattr(intern!(py, "chunks"))?.extract()?; @@ -45,24 +47,25 @@ pub(super) fn from_arrow(obj: &Borrowed<'_, '_, PyAny>) -> PyVortexResult>>()?; let dtype: DType = obj .getattr(intern!(py, "type")) .and_then(|v| DataType::from_pyarrow(&v.as_borrowed())) - .map(|dt| DType::from_arrow(&Field::new("_", dt, false)))?; + .map(|dt| DType::from_arrow_in(&Field::new("_", dt, false), &LEGACY_SESSION))?; Ok(PyArrayRef::from( ChunkedArray::try_new(encoded_chunks, dtype)?.into_array(), )) } else if obj.is_instance(table)? { let array_stream = ArrowArrayStreamReader::from_pyarrow(&obj.as_borrowed())?; - let dtype = DType::from_arrow(array_stream.schema()); + let dtype = DType::from_arrow_in(array_stream.schema(), &LEGACY_SESSION); let chunks = array_stream .into_iter() .map(|b| { b.map_err(VortexError::from) - .and_then(|b| ArrayRef::from_arrow(b, false)) + .and_then(|b| ArrayRef::from_arrow_in(b, false, &LEGACY_SESSION)) }) .collect::>>()?; Ok(PyArrayRef::from( diff --git a/vortex-python/src/dtype/mod.rs b/vortex-python/src/dtype/mod.rs index 1f739e7ab94..b75f6ce0856 100644 --- a/vortex-python/src/dtype/mod.rs +++ b/vortex-python/src/dtype/mod.rs @@ -33,6 +33,7 @@ use pyo3::pyclass; use pyo3::pymethods; use pyo3::types::PyType; use pyo3::wrap_pyfunction; +use vortex::array::LEGACY_SESSION; use vortex::dtype::DType; use vortex::dtype::arrow::FromArrowType; @@ -193,7 +194,10 @@ impl PyDType { ) -> PyResult> { Self::init( cls.py(), - DType::from_arrow(&Field::new("_", arrow_dtype, !non_nullable)), + DType::from_arrow_in( + &Field::new("_", arrow_dtype, !non_nullable), + &LEGACY_SESSION, + ), ) } } diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs index 81bfdc750f6..f360da87482 100644 --- a/vortex-python/src/io.rs +++ b/vortex-python/src/io.rs @@ -11,6 +11,7 @@ use tokio::fs::File; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::arrow::FromArrowArray; use vortex::array::iter::ArrayIterator; use vortex::array::iter::ArrayIteratorAdapter; @@ -436,14 +437,14 @@ fn try_arrow_stream_to_iterator( if ob.is_instance(pa_table)? || ob.is_instance(pa_record_batch_reader)? { // Convert to Arrow stream using FFI let arrow_stream = ArrowArrayStreamReader::from_pyarrow(ob)?; - let dtype = DType::from_arrow(arrow_stream.schema()); + let dtype = DType::from_arrow_in(arrow_stream.schema(), &LEGACY_SESSION); // Convert Arrow RecordBatch stream to Vortex ArrayIterator let vortex_iter = arrow_stream .into_iter() .map(|batch_result| -> VortexResult { let batch = batch_result.map_err(VortexError::from)?; - ArrayRef::from_arrow(batch, false) + ArrayRef::from_arrow_in(batch, false, &LEGACY_SESSION) }); Ok(Box::new(ArrayIteratorAdapter::new(dtype, vortex_iter))) diff --git a/vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs b/vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs index 7cae6944463..b1cc6fcca19 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs @@ -10,6 +10,7 @@ use bytes::Bytes; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::arrays::ChunkedArray; use vortex_array::arrow::FromArrowArray; use vortex_error::VortexResult; @@ -198,7 +199,7 @@ impl DatasetFixture for ClickBenchHits5kFixture { Ok(ChunkedArray::from_iter( batches .into_iter() - .map(|batch| ArrayRef::from_arrow(batch, false)) + .map(|batch| ArrayRef::from_arrow_in(batch, false, &LEGACY_SESSION)) .collect::>>()?, ) .into_array()) diff --git a/vortex-test/compat-gen/src/fixtures/arrays/datasets/tpch.rs b/vortex-test/compat-gen/src/fixtures/arrays/datasets/tpch.rs index 9d831bef143..14b5f8745ec 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/datasets/tpch.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/datasets/tpch.rs @@ -7,6 +7,7 @@ use tpchgen::generators::OrderGenerator; use tpchgen_arrow::RecordBatchIterator; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::arrays::ChunkedArray; use vortex_array::arrow::FromArrowArray; use vortex_error::VortexResult; @@ -20,7 +21,7 @@ fn collect_batches_as_vortex(iter: impl RecordBatchIterator) -> VortexResult>>()?, ) .into_array()) diff --git a/vortex-tui/src/convert.rs b/vortex-tui/src/convert.rs index f6798413422..81a140db705 100644 --- a/vortex-tui/src/convert.rs +++ b/vortex-tui/src/convert.rs @@ -71,13 +71,14 @@ pub async fn exec_convert(session: &VortexSession, flags: ConvertArgs) -> anyhow .with_batch_size(BATCH_SIZE); let num_rows = parquet.metadata().file_metadata().num_rows(); - let dtype = DType::from_arrow(parquet.schema().as_ref()); + let dtype = DType::from_arrow_in(parquet.schema().as_ref(), session); + let session_clone = session.clone(); let mut vortex_stream = parquet .build()? - .map(|record_batch| { + .map(move |record_batch| { record_batch .map_err(|e| vortex_err!(External: e)) - .and_then(|rb| ArrayRef::from_arrow(rb, false)) + .and_then(|rb| ArrayRef::from_arrow_in(rb, false, &session_clone)) }) .boxed(); diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index ae803ee98ae..e729d207ae9 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -226,11 +226,11 @@ mod test { )?)? .build()?; - let dtype = DType::from_arrow(reader.schema()); + let dtype = DType::from_arrow_in(reader.schema(), &LEGACY_SESSION); let chunks: Vec<_> = reader .map(|record_batch| { let batch = record_batch?; - ArrayRef::from_arrow(batch, false) + ArrayRef::from_arrow_in(batch, false, &LEGACY_SESSION) }) .collect::>()?; let vortex_array = ChunkedArray::try_new(chunks, dtype)?.into_array();