From 3bbdd0070728224d2d9259a94385f9e3f43f3496 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 22 Jan 2026 15:07:46 +0000 Subject: [PATCH 1/2] Make flatbuffer writer fallible Signed-off-by: Nicholas Gates --- encodings/runend/src/array.rs | 23 -------- encodings/sequence/src/array.rs | 4 -- encodings/sparse/src/lib.rs | 59 ------------------- .../src/arrays/chunked/vtable/validity.rs | 5 -- .../src/arrays/constant/vtable/validity.rs | 8 --- .../src/arrays/dict/vtable/validity.rs | 36 ----------- vortex-array/src/arrays/filter/vtable.rs | 5 -- vortex-array/src/arrays/masked/array.rs | 6 -- vortex-array/src/arrays/masked/tests.rs | 2 +- vortex-array/src/arrays/masked/vtable/mod.rs | 2 +- vortex-array/src/arrays/null/mod.rs | 5 -- .../src/arrays/scalar_fn/vtable/validity.rs | 11 ---- vortex-array/src/arrays/slice/vtable.rs | 5 -- vortex-array/src/arrow/array.rs | 9 --- vortex-array/src/vtable/validity.rs | 9 --- vortex-python/src/arrays/py/vtable.rs | 5 -- 16 files changed, 2 insertions(+), 192 deletions(-) diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 61edf670e64..0e33f9097a3 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -15,7 +15,6 @@ use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; -use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveVTable; use vortex_array::buffer::BufferHandle; use vortex_array::search_sorted::SearchSorted; @@ -38,7 +37,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; -use vortex_mask::Mask; use vortex_scalar::PValue; use crate::compress::runend_decode_bools; @@ -455,27 +453,6 @@ impl ValidityVTable for RunEndVTable { }), }) } - - fn validity_mask(array: &RunEndArray) -> VortexResult { - Ok(match array.values().validity_mask()? { - Mask::AllTrue(_) => Mask::AllTrue(array.len()), - Mask::AllFalse(_) => Mask::AllFalse(array.len()), - Mask::Values(values) => { - // SAFETY: we preserve ends from an existing validated RunEndArray. - // Validity is checked on construction to have the correct len. - let ree_validity = unsafe { - RunEndArray::new_unchecked( - array.ends().clone(), - values.into_array(), - array.offset(), - array.len(), - ) - .into_array() - }; - Mask::from_buffer(ree_validity.to_bool().bit_buffer().clone()) - } - }) - } } pub(super) fn run_end_canonicalize( diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index fad4ead18d7..16b32511c24 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -400,10 +400,6 @@ impl ValidityVTable for SequenceVTable { fn validity(_array: &SequenceArray) -> VortexResult { Ok(Validity::AllValid) } - - fn validity_mask(array: &SequenceArray) -> VortexResult { - Ok(Mask::AllTrue(array.len())) - } } impl VisitorVTable for SequenceVTable { diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 69e4cfd7064..7167ef3f622 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -5,7 +5,6 @@ use std::fmt::Debug; use std::hash::Hash; use std::ops::Range; -use itertools::Itertools as _; use num_traits::AsPrimitive; use prost::Message as _; use vortex_array::Array; @@ -40,13 +39,11 @@ use vortex_array::vtable::NotSupported; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; use vortex_array::vtable::VisitorVTable; -use vortex_buffer::BitBufferMut; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; use vortex_dtype::DType; use vortex_dtype::NativePType; use vortex_dtype::Nullability; -use vortex_dtype::match_each_integer_ptype; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -429,62 +426,6 @@ impl ValidityVTable for SparseVTable { .into_array(), )) } - - fn validity_mask(array: &SparseArray) -> VortexResult { - let fill_is_valid = array.fill_scalar().is_valid(); - let values_validity = array.patches().values().validity_mask()?; - let len = array.len(); - - if matches!(values_validity, Mask::AllTrue(_)) && fill_is_valid { - return Ok(Mask::AllTrue(len)); - } - if matches!(values_validity, Mask::AllFalse(_)) && !fill_is_valid { - return Ok(Mask::AllFalse(len)); - } - - let mut is_valid_buffer = if fill_is_valid { - BitBufferMut::new_set(len) - } else { - BitBufferMut::new_unset(len) - }; - - let indices = array.patches().indices().to_primitive(); - let index_offset = array.patches().offset(); - - match_each_integer_ptype!(indices.ptype(), |I| { - let indices = indices.as_slice::(); - patch_validity(&mut is_valid_buffer, indices, index_offset, values_validity); - }); - - Ok(Mask::from_buffer(is_valid_buffer.freeze())) - } -} - -fn patch_validity>( - is_valid_buffer: &mut BitBufferMut, - indices: &[I], - index_offset: usize, - values_validity: Mask, -) { - let indices = indices.iter().map(|index| index.as_() - index_offset); - match values_validity { - Mask::AllTrue(_) => { - for index in indices { - is_valid_buffer.set(index); - } - } - Mask::AllFalse(_) => { - for index in indices { - is_valid_buffer.unset(index); - } - } - Mask::Values(mask_values) => { - let is_valid = mask_values.bit_buffer().iter(); - for (index, is_valid) in indices.zip_eq(is_valid) { - is_valid_buffer.set_to(index, is_valid); - } - } - } } impl VisitorVTable for SparseVTable { diff --git a/vortex-array/src/arrays/chunked/vtable/validity.rs b/vortex-array/src/arrays/chunked/vtable/validity.rs index fee95a4be20..10e3f947200 100644 --- a/vortex-array/src/arrays/chunked/vtable/validity.rs +++ b/vortex-array/src/arrays/chunked/vtable/validity.rs @@ -5,7 +5,6 @@ use itertools::Itertools; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_error::VortexResult; -use vortex_mask::Mask; use crate::Array; use crate::IntoArray; @@ -52,8 +51,4 @@ impl ValidityVTable for ChunkedVTable { .into_array(), )) } - - fn validity_mask(array: &ChunkedArray) -> VortexResult { - array.chunks().iter().map(|a| a.validity_mask()).collect() - } } diff --git a/vortex-array/src/arrays/constant/vtable/validity.rs b/vortex-array/src/arrays/constant/vtable/validity.rs index c89ac0054da..c09449f0ec0 100644 --- a/vortex-array/src/arrays/constant/vtable/validity.rs +++ b/vortex-array/src/arrays/constant/vtable/validity.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_error::VortexResult; -use vortex_mask::Mask; use crate::arrays::ConstantArray; use crate::arrays::ConstantVTable; @@ -17,11 +16,4 @@ impl ValidityVTable for ConstantVTable { Validity::AllValid }) } - - fn validity_mask(array: &ConstantArray) -> VortexResult { - Ok(match array.scalar().is_null() { - true => Mask::AllFalse(array.len), - false => Mask::AllTrue(array.len), - }) - } } diff --git a/vortex-array/src/arrays/dict/vtable/validity.rs b/vortex-array/src/arrays/dict/vtable/validity.rs index 6545243fcb8..1efb139830f 100644 --- a/vortex-array/src/arrays/dict/vtable/validity.rs +++ b/vortex-array/src/arrays/dict/vtable/validity.rs @@ -1,18 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_buffer::BitBuffer; use vortex_dtype::Nullability; -use vortex_dtype::match_each_integer_ptype; use vortex_error::VortexResult; -use vortex_mask::AllOr; -use vortex_mask::Mask; use vortex_scalar::Scalar; use super::DictVTable; use crate::Array; use crate::IntoArray; -use crate::ToCanonical; use crate::arrays::dict::DictArray; use crate::compute::fill_null; use crate::validity::Validity; @@ -54,35 +49,4 @@ impl ValidityVTable for DictVTable { }, ) } - - fn validity_mask(array: &DictArray) -> VortexResult { - let codes_validity = array.codes().validity_mask()?; - Ok(match codes_validity.bit_buffer() { - AllOr::All => { - let primitive_codes = array.codes().to_primitive(); - let values_mask = array.values().validity_mask()?; - let is_valid_buffer = match_each_integer_ptype!(primitive_codes.ptype(), |P| { - let codes_slice = primitive_codes.as_slice::

(); - BitBuffer::collect_bool(array.len(), |idx| { - #[allow(clippy::cast_possible_truncation)] - values_mask.value(codes_slice[idx] as usize) - }) - }); - Mask::from_buffer(is_valid_buffer) - } - AllOr::None => Mask::AllFalse(array.len()), - AllOr::Some(validity_buff) => { - let primitive_codes = array.codes().to_primitive(); - let values_mask = array.values().validity_mask()?; - let is_valid_buffer = match_each_integer_ptype!(primitive_codes.ptype(), |P| { - let codes_slice = primitive_codes.as_slice::

(); - #[allow(clippy::cast_possible_truncation)] - BitBuffer::collect_bool(array.len(), |idx| { - validity_buff.value(idx) && values_mask.value(codes_slice[idx] as usize) - }) - }); - Mask::from_buffer(is_valid_buffer) - } - }) - } } diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index c16a7b67a98..5114f0b3f1c 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -5,7 +5,6 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::hash::Hasher; -use vortex_compute::filter::Filter; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -200,10 +199,6 @@ impl ValidityVTable for FilterVTable { fn validity(array: &FilterArray) -> VortexResult { array.child.validity()?.filter(&array.mask) } - - fn validity_mask(array: &FilterArray) -> VortexResult { - Ok(Filter::filter(&array.child.validity_mask()?, &array.mask)) - } } impl VisitorVTable for FilterVTable { diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index abb51e0ea20..3468e99b3aa 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -4,7 +4,6 @@ use vortex_dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_mask::Mask; use crate::ArrayRef; use crate::stats::ArrayStats; @@ -49,9 +48,4 @@ impl MaskedArray { pub fn child(&self) -> &ArrayRef { &self.child } - - /// Get the validity mask for this array. - pub fn validity_mask(&self) -> Mask { - self.validity.to_mask(self.len()) - } } diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index fb0512bcd9e..5a182ed5175 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -97,5 +97,5 @@ fn test_masked_child_preserves_length(#[case] validity: Validity) { let array = MaskedArray::try_new(child, validity.clone()).unwrap(); assert_eq!(array.len(), len); - assert_eq!(array.validity_mask(), validity.to_mask(len)); + assert_eq!(array.validity_mask().unwrap(), validity.to_mask(len)); } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index e8def1d8b18..45773a8656c 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -125,7 +125,7 @@ impl VTable for MaskedVTable { } fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - let validity_mask = array.validity_mask(); + let validity_mask = array.validity_mask()?; // Fast path: all masked means result is all nulls. if validity_mask.all_false() { diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index fcd2436c916..722ef0ce5ff 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -7,7 +7,6 @@ use std::ops::Range; use vortex_dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_ensure; -use vortex_mask::Mask; use vortex_scalar::Scalar; use crate::ArrayBufferVisitor; @@ -175,8 +174,4 @@ impl ValidityVTable for NullVTable { fn validity(_array: &NullArray) -> VortexResult { Ok(Validity::AllInvalid) } - - fn validity_mask(array: &NullArray) -> VortexResult { - Ok(Mask::AllFalse(array.len)) - } } diff --git a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs index 97c9a7c2ba4..9156eeef3fd 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use vortex_error::VortexResult; -use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; @@ -14,7 +13,6 @@ use crate::arrays::scalar_fn::array::ScalarFnArray; use crate::arrays::scalar_fn::vtable::ArrayExpr; use crate::arrays::scalar_fn::vtable::FakeEq; use crate::arrays::scalar_fn::vtable::ScalarFnVTable; -use crate::executor::CanonicalOutput; use crate::expr::ExecutionArgs; use crate::expr::Expression; use crate::expr::Literal; @@ -76,13 +74,4 @@ impl ValidityVTable for ScalarFnVTable { // Execute the validity expression. All leaves are ArrayExpr nodes. Ok(Validity::Array(execute_expr(&validity_expr, array.len())?)) } - - fn validity_mask(array: &ScalarFnArray) -> VortexResult { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let output = array.to_array().execute::(&mut ctx)?; - Ok(match output { - CanonicalOutput::Constant(c) => Mask::new(array.len, c.scalar().is_valid()), - CanonicalOutput::Array(a) => a.into_array().validity()?.to_mask(array.len()), - }) - } } diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index 95e2969d79f..0b328420ea1 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -12,7 +12,6 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; -use vortex_mask::Mask; use vortex_scalar::Scalar; use crate::Array; @@ -166,10 +165,6 @@ impl ValidityVTable for SliceVTable { fn validity(array: &SliceArray) -> VortexResult { Ok(array.child.validity()?.slice(array.range.clone())) } - - fn validity_mask(array: &SliceArray) -> VortexResult { - Ok(array.child.validity_mask()?.slice(array.range.clone())) - } } impl VisitorVTable for SliceVTable { diff --git a/vortex-array/src/arrow/array.rs b/vortex-array/src/arrow/array.rs index 8df97b55ccf..c991baa66b3 100644 --- a/vortex-array/src/arrow/array.rs +++ b/vortex-array/src/arrow/array.rs @@ -12,7 +12,6 @@ use vortex_dtype::arrow::FromArrowType; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; -use vortex_mask::Mask; use vortex_scalar::Scalar; use crate::ArrayBufferVisitor; @@ -171,14 +170,6 @@ impl ValidityVTable for ArrowVTable { }, }) } - - fn validity_mask(array: &ArrowArray) -> VortexResult { - Ok(array - .inner - .logical_nulls() - .map(|null_buffer| Mask::from_buffer(null_buffer.inner().clone().into())) - .unwrap_or_else(|| Mask::new_true(array.inner.len()))) - } } impl VisitorVTable for ArrowVTable { diff --git a/vortex-array/src/vtable/validity.rs b/vortex-array/src/vtable/validity.rs index 89e517365ba..5ece2786ec6 100644 --- a/vortex-array/src/vtable/validity.rs +++ b/vortex-array/src/vtable/validity.rs @@ -3,7 +3,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; -use vortex_mask::Mask; use crate::Array; use crate::ArrayRef; @@ -18,10 +17,6 @@ pub trait ValidityVTable { /// /// - The array DType is nullable. fn validity(array: &V::Array) -> VortexResult; - - fn validity_mask(array: &V::Array) -> VortexResult { - Ok(Self::validity(array)?.to_mask(array.len())) - } } impl ValidityVTable for NotSupported { @@ -87,10 +82,6 @@ where fn validity(array: &V::Array) -> VortexResult { V::validity_child(array).validity() } - - fn validity_mask(array: &V::Array) -> VortexResult { - V::validity_child(array).validity_mask() - } } /// An implementation of the [`ValidityVTable`] for arrays that hold an unsliced validity diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index 8e65e50993e..ddae074b026 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -35,7 +35,6 @@ use vortex::dtype::DType; use vortex::error::VortexResult; use vortex::error::vortex_ensure; use vortex::error::vortex_err; -use vortex::mask::Mask; use vortex::scalar::Scalar; use crate::arrays::py::PythonArray; @@ -151,10 +150,6 @@ impl ValidityVTable for PythonVTable { fn validity(_array: &PythonArray) -> VortexResult { todo!() } - - fn validity_mask(_array: &PythonArray) -> VortexResult { - todo!() - } } impl VisitorVTable for PythonVTable { From 0b5fa2ea88413d1b60d11090b8c35bf33661d64b Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 22 Jan 2026 15:38:28 +0000 Subject: [PATCH 2/2] merge Signed-off-by: Nicholas Gates --- encodings/sparse/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 7167ef3f622..b590006b1a3 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -5,7 +5,6 @@ use std::fmt::Debug; use std::hash::Hash; use std::ops::Range; -use num_traits::AsPrimitive; use prost::Message as _; use vortex_array::Array; use vortex_array::ArrayBufferVisitor; @@ -42,7 +41,6 @@ use vortex_array::vtable::VisitorVTable; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; use vortex_dtype::DType; -use vortex_dtype::NativePType; use vortex_dtype::Nullability; use vortex_error::VortexExpect as _; use vortex_error::VortexResult;