fix scalar equality ignores nullability

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>

Author: connortsui20

vortex-array/src/arrays/primitive/compute/sum.rs

@@ -25,7 +25,7 @@ use crate::register_kernel;
 /// Helper to convert an Option<T> to a Scalar. None represents overflow (null result).
 fn option_to_scalar<T: NativePType + Into<PValue>>(opt: Option<T>) -> Scalar {
     match opt {
-        Some(v) => Scalar::primitive(v, Nullability::NonNullable),
+        Some(v) => Scalar::primitive(v, Nullability::Nullable),
         None => Scalar::null(T::PTYPE.into()),
     }
 }
@@ -55,7 +55,7 @@ impl SumKernel for PrimitiveVTable {
                                 array.as_slice::<T>(),
                                 accumulator.as_primitive().as_::<f64>().vortex_expect("cannot be null"),
                             ),
-                            Nullability::NonNullable,
+                            Nullability::Nullable,
                         )
                     }
                 )
@@ -89,7 +89,7 @@ impl SumKernel for PrimitiveVTable {
                                 validity_mask,
                                 accumulator.as_primitive().as_::<f64>().vortex_expect("cannot be null"),
                             ),
-                            Nullability::NonNullable,
+                            Nullability::Nullable,
                         )
                     }
                 )

vortex-scalar/src/scalar.rs

@@ -2,6 +2,8 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::cmp::Ordering;
+use std::hash::Hash;
+use std::hash::Hasher;
 
 use vortex_dtype::DType;
 use vortex_dtype::NativeDType;
@@ -27,7 +29,7 @@ use crate::Utf8Scalar;
 ///
 /// Scalars represent a single value with an associated [`DType`]. The value can be null, in which
 /// case the [`value`][Scalar::value] method returns `None`.
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Clone, Debug, Eq)]
 pub struct Scalar {
     /// The type of the scalar.
     pub(crate) dtype: DType,
@@ -38,6 +40,19 @@ pub struct Scalar {
     pub(crate) value: Option<ScalarValue>,
 }
 
+impl PartialEq for Scalar {
+    fn eq(&self, other: &Self) -> bool {
+        self.dtype.eq_ignore_nullability(&other.dtype) && self.value == other.value
+    }
+}
+
+impl Hash for Scalar {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.dtype.as_nonnullable().hash(state);
+        self.value.hash(state);
+    }
+}
+
 impl Scalar {
     // Constructors for null scalars.