ExtScalar VTables

encodings/datetime-parts/src/ops.rs

@@ -62,7 +62,7 @@ impl OperationsVTable<DateTimePartsVTable> for DateTimePartsVTable {
 
         Ok(Scalar::extension::<Timestamp>(
             options.clone(),
-            Scalar::primitive(ts, ext.storage_dtype().nullability()),
+            Scalar::primitive(ts, ext.nullability()),
         ))
     }
 }

vortex-array/src/array/mod.rs

@@ -23,6 +23,7 @@ use vortex_error::vortex_err;
 use vortex_error::vortex_panic;
 use vortex_mask::Mask;
 use vortex_scalar::Scalar;
+use vortex_scalar::ScalarValue;
 
 use crate::ArrayEq;
 use crate::ArrayHash;
@@ -510,11 +511,9 @@ impl<V: VTable> Array for ArrayAdapter<V> {
                     matches!(
                         stat,
                         Stat::IsConstant | Stat::IsSorted | Stat::IsStrictSorted
-                    ) && value.as_ref().as_exact().is_some_and(|v| {
-                        Scalar::new(DType::Bool(Nullability::NonNullable), v.clone())
-                            .as_bool()
-                            .value()
-                            .unwrap_or_default()
+                    ) && value.as_ref().as_exact().is_some_and(|v| match v {
+                        ScalarValue::Bool(b) => *b,
+                        _ => vortex_panic!("Unexpected scalar value type in stats propagation"),
                     })
                 }));
             });

vortex-array/src/arrays/constant/compute/take.rs

@@ -20,13 +20,12 @@ impl TakeKernel for ConstantVTable {
     fn take(&self, array: &ConstantArray, indices: &dyn Array) -> VortexResult<ArrayRef> {
         match indices.validity_mask()?.bit_buffer() {
             AllOr::All => {
-                let scalar = Scalar::new(
-                    array
+                let scalar = array.scalar().cast(
+                    &array
                         .scalar()
                         .dtype()
                         .union_nullability(indices.dtype().nullability()),
-                    array.scalar().value().clone(),
-                );
+                )?;
                 Ok(ConstantArray::new(scalar, indices.len()).into_array())
             }
             AllOr::None => Ok(ConstantArray::new(

vortex-array/src/arrays/extension/vtable/rules.rs

@@ -82,7 +82,11 @@ mod tests {
             ExtID::new_ref("test_ext")
         }
 
-        fn validate(&self, _options: &Self::Metadata, _storage_dtype: &DType) -> VortexResult<()> {
+        fn validate_dtype(
+            &self,
+            _options: &Self::Metadata,
+            _storage_dtype: &DType,
+        ) -> VortexResult<()> {
             Ok(())
         }
     }
@@ -162,7 +166,7 @@ mod tests {
                 ExtID::new_ref("test_ext_2")
             }
 
-            fn validate(
+            fn validate_dtype(
                 &self,
                 _options: &Self::Metadata,
                 _storage_dtype: &DType,

vortex-array/src/builders/bool.rs

@@ -11,7 +11,6 @@ use vortex_error::VortexExpect;
 use vortex_error::VortexResult;
 use vortex_error::vortex_ensure;
 use vortex_mask::Mask;
-use vortex_scalar::BoolScalar;
 use vortex_scalar::Scalar;
 
 use crate::Array;
@@ -105,7 +104,7 @@ impl ArrayBuilder for BoolBuilder {
             scalar.dtype()
         );
 
-        let bool_scalar = BoolScalar::try_from(scalar)?;
+        let bool_scalar = scalar.as_bool();
         match bool_scalar.value() {
             Some(value) => self.append_value(value),
             None => self.append_null(),

vortex-array/src/builders/decimal.rs

@@ -180,7 +180,7 @@ impl ArrayBuilder for DecimalBuilder {
         match scalar.as_decimal().decimal_value() {
             None => self.append_null(),
             Some(v) => match_each_decimal_value!(v, |dec_val| {
-                self.append_value(dec_val);
+                self.append_value(*dec_val);
             }),
         }
 

vortex-array/src/builders/extension.rs

@@ -8,8 +8,8 @@ use vortex_dtype::extension::ExtDTypeRef;
 use vortex_error::VortexResult;
 use vortex_error::vortex_ensure;
 use vortex_mask::Mask;
-use vortex_scalar::ExtScalar;
 use vortex_scalar::Scalar;
+use vortex_scalar::extension::ExtensionScalar;
 
 use crate::Array;
 use crate::ArrayRef;
@@ -42,8 +42,9 @@ impl ExtensionBuilder {
     }
 
     /// Appends an extension `value` to the builder.
-    pub fn append_value(&mut self, value: ExtScalar) -> VortexResult<()> {
-        self.storage.append_scalar(&value.storage())
+    pub fn append_value(&mut self, scalar: ExtensionScalar) -> VortexResult<()> {
+        let storage_scalar = scalar.to_storage_scalar()?;
+        self.storage.append_scalar(&storage_scalar)
     }
 
     /// Finishes the builder directly into a [`ExtensionArray`].
@@ -94,9 +95,7 @@ impl ArrayBuilder for ExtensionBuilder {
             self.dtype(),
             scalar.dtype()
         );
-
-        let ext_scalar = ExtScalar::try_from(scalar)?;
-        self.append_value(ext_scalar)
+        self.append_value(scalar.as_extension())
     }
 
     unsafe fn extend_from_array_unchecked(&mut self, array: &dyn Array) {

vortex-array/src/builders/fixed_size_list.rs

@@ -8,11 +8,10 @@ use vortex_dtype::DType;
 use vortex_dtype::Nullability;
 use vortex_error::VortexExpect;
 use vortex_error::VortexResult;
-use vortex_error::vortex_bail;
 use vortex_error::vortex_ensure;
 use vortex_error::vortex_panic;
 use vortex_mask::Mask;
-use vortex_scalar::ListScalar;
+use vortex_scalar::FixedSizeListScalar;
 use vortex_scalar::Scalar;
 
 use crate::Array;
@@ -101,31 +100,23 @@ impl FixedSizeListBuilder {
     }
 
     /// Appends a fixed-size list `value` to the builder.
-    ///
-    /// Note that a [`ListScalar`] can represent both a [`ListArray`] scalar **and** a
-    /// [`FixedSizeListArray`] scalar (since a single list cannot know the size of other lists in
-    /// fixed-size list arrays without accompanying metadata).
-    ///
-    /// [`ListArray`]: crate::arrays::ListArray
-    pub fn append_value(&mut self, value: ListScalar) -> VortexResult<()> {
-        let Some(elements) = value.elements() else {
+    pub fn append_value(&mut self, value: FixedSizeListScalar) -> VortexResult<()> {
+        let Some(elements) = value.elements_iter() else {
             // If `elements` is `None`, then the `value` is a null value.
             self.append_null();
             return Ok(());
         };
 
-        if value.len() != self.list_size() as usize {
-            vortex_bail!(
-                "Tried to append a `ListScalar` with length {} to a `FixedSizeListScalar` \
-                    with fixed size of {}",
-                value.len(),
-                self.list_size()
-            );
-        }
+        vortex_ensure!(
+            value.list_size() == self.list_size(),
+            "Tried to append a `FixedSizeListScalar` with list size {} to a `FixedSizeListScalar` with fixed size of {}",
+            value.list_size(),
+            self.list_size()
+        );
 
         for scalar in elements {
             // TODO(connor): This is slow, we should be able to append multiple values at once, or
-            // the list scalar should hold an Array
+            //  the list scalar should hold an Array
             self.elements_builder.append_scalar(&scalar)?;
         }
         self.nulls.append_non_null();
@@ -228,7 +219,7 @@ impl ArrayBuilder for FixedSizeListBuilder {
             scalar.dtype()
         );
 
-        let list_scalar = scalar.as_list();
+        let list_scalar = scalar.as_fixed_size_list();
         self.append_value(list_scalar)
     }
 
@@ -310,7 +301,7 @@ mod tests {
                     vec![1i32.into(), 2i32.into(), 3i32.into()],
                     NonNullable,
                 )
-                .as_list(),
+                .as_fixed_size_list(),
             )
             .unwrap();
 
@@ -321,7 +312,7 @@ mod tests {
                     vec![4i32.into(), 5i32.into(), 6i32.into()],
                     NonNullable,
                 )
-                .as_list(),
+                .as_fixed_size_list(),
             )
             .unwrap();
 
@@ -342,7 +333,10 @@ mod tests {
         // Append multiple "empty" lists.
         for _ in 0..100 {
             builder
-                .append_value(Scalar::fixed_size_list(dtype.clone(), vec![], NonNullable).as_list())
+                .append_value(
+                    Scalar::fixed_size_list(dtype.clone(), vec![], NonNullable)
+                        .as_fixed_size_list(),
+                )
                 .unwrap();
         }
 
@@ -366,7 +360,8 @@ mod tests {
             if i % 2 == 0 {
                 builder
                     .append_value(
-                        Scalar::fixed_size_list(dtype.clone(), vec![], Nullable).as_list(),
+                        Scalar::fixed_size_list(dtype.clone(), vec![], Nullable)
+                            .as_fixed_size_list(),
                     )
                     .unwrap();
             } else {
@@ -397,7 +392,7 @@ mod tests {
                         vec![(i * 2).into(), (i * 2 + 1).into()],
                         NonNullable,
                     )
-                    .as_list(),
+                    .as_fixed_size_list(),
                 )
                 .unwrap();
         }
@@ -431,15 +426,16 @@ mod tests {
         builder
             .append_value(
                 Scalar::fixed_size_list(dtype.clone(), vec![1i32.into(), 2i32.into()], Nullable)
-                    .as_list(),
+                    .as_fixed_size_list(),
             )
             .unwrap();
 
         builder.append_null();
 
         builder
             .append_value(
-                Scalar::fixed_size_list(dtype, vec![3i32.into(), 4i32.into()], Nullable).as_list(),
+                Scalar::fixed_size_list(dtype, vec![3i32.into(), 4i32.into()], Nullable)
+                    .as_fixed_size_list(),
             )
             .unwrap();
 
@@ -468,7 +464,7 @@ mod tests {
                     ],
                     NonNullable,
                 )
-                .as_list(),
+                .as_fixed_size_list(),
             )
             .unwrap();
 
@@ -483,7 +479,7 @@ mod tests {
                     ],
                     NonNullable,
                 )
-                .as_list(),
+                .as_fixed_size_list(),
             )
             .unwrap();
 
@@ -587,7 +583,7 @@ mod tests {
                 vec![1i32.into(), 2i32.into()], // Only 2 elements, not 3.
                 NonNullable,
             )
-            .as_list(),
+            .as_fixed_size_list(),
         );
 
         assert!(result.is_err());
@@ -693,7 +689,7 @@ mod tests {
                     vec![1i32.into(), 2i32.into(), 3i32.into()],
                     NonNullable,
                 )
-                .as_list(),
+                .as_fixed_size_list(),
             )
             .unwrap();
 

vortex-array/src/expr/exprs/dynamic.rs

@@ -105,11 +105,11 @@ impl VTable for DynamicComparison {
                 ctx: args.ctx,
             });
         }
-        let ret_dtype =
-            DType::Bool(args.inputs[0].dtype().nullability() | data.rhs.dtype.nullability());
+
+        let nullability = args.inputs[0].dtype().nullability();
 
         Ok(ExecutionResult::Scalar(ConstantArray::new(
-            Scalar::new(ret_dtype, data.default.into()),
+            Scalar::bool(data.default, nullability),
             args.row_count,
         )))
     }
@@ -194,7 +194,9 @@ pub struct DynamicComparisonExpr {
 
 impl DynamicComparisonExpr {
     pub fn scalar(&self) -> Option<Scalar> {
-        (self.rhs.value)().map(|v| Scalar::new(self.rhs.dtype.clone(), v))
+        (self.rhs.value)().map(|v| {
+            Scalar::try_new(self.rhs.dtype.clone(), v).vortex_expect("Rhs value should match dtype")
+        })
     }
 }
 
@@ -238,7 +240,9 @@ struct Rhs {
 
 impl Rhs {
     pub fn scalar(&self) -> Option<Scalar> {
-        (self.value)().map(|v| Scalar::new(self.dtype.clone(), v))
+        (self.value)().map(|v| {
+            Scalar::try_new(self.dtype.clone(), v).vortex_expect("Rhs value should match dtype")
+        })
     }
 }
 
@@ -284,7 +288,12 @@ impl DynamicExprUpdates {
         let exprs = visitor.0.into_boxed_slice();
         let prev_versions = exprs
             .iter()
-            .map(|expr| (expr.rhs.value)().map(|v| Scalar::new(expr.rhs.dtype.clone(), v)))
+            .map(|expr| {
+                (expr.rhs.value)().map(|v| {
+                    Scalar::try_new(expr.rhs.dtype.clone(), v)
+                        .vortex_expect("Rhs value should match dtype")
+                })
+            })
             .collect();
 
         Some(Self {

vortex-array/src/expr/exprs/like.rs

@@ -155,10 +155,10 @@ impl VTable for Like {
         }
 
         // Extract the pattern out
-        let pat = expr.child(1).as_::<Literal>();
+        let pat = expr.child(1).as_::<Literal>().as_utf8();
 
         // LIKE NULL is nonsensical, don't try to handle it
-        let pat_str = pat.as_utf8().value()?;
+        let pat_str = pat.value()?;
 
         let src = expr.child(0).clone();
         let src_min = src.stat_min(catalog)?;

vortex-array/src/expr/exprs/literal.rs

@@ -36,7 +36,7 @@ impl VTable for Literal {
     fn serialize(&self, instance: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
         Ok(Some(
             pb::LiteralOpts {
-                value: Some(instance.as_ref().into()),
+                value: Some(instance.into()),
             }
             .encode_to_vec(),
         ))

vortex-dtype/src/datetime/date.rs

@@ -15,7 +15,7 @@ use crate::extension::ExtDTypeVTable;
 use crate::extension::ExtID;
 
 /// Date DType.
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
 pub struct Date;
 
 impl Date {
@@ -54,7 +54,7 @@ impl ExtDTypeVTable for Date {
         TimeUnit::try_from(tag)
     }
 
-    fn validate(&self, metadata: &Self::Metadata, storage_dtype: &DType) -> VortexResult<()> {
+    fn validate_dtype(&self, metadata: &Self::Metadata, storage_dtype: &DType) -> VortexResult<()> {
         let ptype = date_ptype(metadata)
             .ok_or_else(|| vortex_err!("Date type does not support time unit {}", metadata))?;