diff --git a/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs b/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs index 6b2feb380a7..1d04a074d6f 100644 --- a/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs @@ -289,7 +289,7 @@ fn supports_uncompressed_size_in_bytes(dtype: &DType) -> bool { DType::Struct(fields, _) => fields .fields() .all(|field| supports_uncompressed_size_in_bytes(&field)), - DType::Union(variants) => variants + DType::Union(variants, _) => variants .variants() .all(|variant| supports_uncompressed_size_in_bytes(&variant)), DType::Variant(_) => false, diff --git a/vortex-array/src/dtype/coercion.rs b/vortex-array/src/dtype/coercion.rs index 3df356d93af..4d30a80c216 100644 --- a/vortex-array/src/dtype/coercion.rs +++ b/vortex-array/src/dtype/coercion.rs @@ -79,23 +79,8 @@ impl DType { /// The core primitive — what type can hold both `self` and `other`? /// Returns `None` if no common supertype exists. pub fn least_supertype(&self, other: &DType) -> Option { - match (self, other) { - (DType::Union(lhs), DType::Union(rhs)) => { - return (lhs == rhs).then(|| self.clone()); - } - (DType::Null, DType::Union(variants)) => { - return variants - .derived_nullability() - .is_nullable() - .then(|| other.clone()); - } - (DType::Union(variants), DType::Null) => { - return variants - .derived_nullability() - .is_nullable() - .then(|| self.clone()); - } - _ => {} + if let (DType::Union(lhs, lhs_null), DType::Union(rhs, rhs_null)) = (self, other) { + return (lhs == rhs).then(|| DType::Union(lhs.clone(), *lhs_null | *rhs_null)); } let union_null = self.nullability() | other.nullability(); @@ -377,6 +362,7 @@ mod tests { vec![DType::Primitive(PType::I32, NonNullable)], ) .unwrap(), + NonNullable, ); let nullable = DType::Union( UnionVariants::new( @@ -384,6 +370,7 @@ mod tests { vec![DType::Primitive(PType::I32, Nullable)], ) .unwrap(), + NonNullable, ); assert_eq!( @@ -395,29 +382,22 @@ mod tests { } #[test] - fn least_supertype_null_requires_nullable_union() { + fn least_supertype_null_makes_union_outer_nullable() { let nonnullable = DType::Union( UnionVariants::new( ["value"].into(), vec![DType::Primitive(PType::I32, NonNullable)], ) .unwrap(), + NonNullable, ); - let nullable = DType::Union( - UnionVariants::new( - ["value"].into(), - vec![DType::Primitive(PType::I32, Nullable)], - ) - .unwrap(), - ); + let nullable = nonnullable.as_nullable(); - assert!(DType::Null.least_supertype(&nonnullable).is_none()); - assert!(nonnullable.least_supertype(&DType::Null).is_none()); assert_eq!( - DType::Null.least_supertype(&nullable), + DType::Null.least_supertype(&nonnullable), Some(nullable.clone()) ); - assert_eq!(nullable.least_supertype(&DType::Null), Some(nullable)); + assert_eq!(nonnullable.least_supertype(&DType::Null), Some(nullable)); } #[test] diff --git a/vortex-array/src/dtype/dtype_impl.rs b/vortex-array/src/dtype/dtype_impl.rs index 5767a124667..20ff3d6a182 100644 --- a/vortex-array/src/dtype/dtype_impl.rs +++ b/vortex-array/src/dtype/dtype_impl.rs @@ -64,8 +64,8 @@ impl DType { | List(_, null) | FixedSizeList(_, _, null) | Struct(_, null) + | Union(_, null) | Variant(null) => matches!(null, Nullability::Nullable), - Union(variants) => variants.derived_nullability().is_nullable(), Extension(ext_dtype) => ext_dtype.storage_dtype().is_nullable(), } } @@ -82,8 +82,7 @@ impl DType { /// Get a new DType with the given nullability (but otherwise the same as `self`). /// - /// [`DType::Null`] and [`DType::Union`] have intrinsic nullability and are returned unchanged. - /// To change a union's nullability, construct different [`UnionVariants`]. + /// [`DType::Null`] has intrinsic nullability and is returned unchanged. pub fn with_nullability(&self, nullability: Nullability) -> Self { match self { Null => Null, @@ -95,7 +94,7 @@ impl DType { List(edt, _) => List(Arc::clone(edt), nullability), FixedSizeList(edt, size, _) => FixedSizeList(Arc::clone(edt), *size, nullability), Struct(sf, _) => Struct(sf.clone(), nullability), - Union(vs) => Union(vs.clone()), + Union(vs, _) => Union(vs.clone(), nullability), Variant(_) => Variant(nullability), Extension(ext) => Extension(ext.with_nullability(nullability)), } @@ -127,7 +126,7 @@ impl DType { .zip_eq(rhs_dtype.fields()) .all(|(l, r)| l.eq_ignore_nullability(&r))) } - (Union(lhs), Union(rhs)) => { + (Union(lhs, _), Union(rhs, _)) => { // Equal `names` implies equal length by FieldNames equality. lhs.names() == rhs.names() && lhs.type_ids() == rhs.type_ids() @@ -439,12 +438,20 @@ impl DType { /// Get the [`UnionVariants`] if `self` is a [`DType::Union`], otherwise `None`. pub fn as_union_variants_opt(&self) -> Option<&UnionVariants> { - if let Union(uv) = self { Some(uv) } else { None } + if let Union(uv, _) = self { + Some(uv) + } else { + None + } } /// Owned version of [Self::as_union_variants_opt]. pub fn into_union_variants_opt(self) -> Option { - if let Union(uv) = self { Some(uv) } else { None } + if let Union(uv, _) = self { + Some(uv) + } else { + None + } } /// Downcast a `DType` to an `ExtDType` @@ -498,7 +505,7 @@ impl Display for DType { .map(|(field_null, dt)| format!("{field_null}={dt}")) .join(", "), ), - Union(uv) => write!(f, "union({uv}){}", uv.derived_nullability()), + Union(uv, null) => write!(f, "union({uv}){null}"), Variant(null) => write!(f, "variant{null}"), Extension(ext) => write!(f, "{}", ext), } diff --git a/vortex-array/src/dtype/mod.rs b/vortex-array/src/dtype/mod.rs index 0160b4c36c7..8b579c70d39 100644 --- a/vortex-array/src/dtype/mod.rs +++ b/vortex-array/src/dtype/mod.rs @@ -111,15 +111,13 @@ pub enum DType { /// A logical union (sum) type. /// /// A `Union` is composed of one or more **variants**, each with a name and a `DType`. A per-row - /// `i8` tag selects which variant is "live" at that row. + /// `u8` tag selects which variant is "live" at that row. /// - /// Unlike other nested types, a union has no independent outer nullability. Its nullability is - /// derived at runtime from its variants: a union is nullable when any variant can contain null. - /// A concrete union array has no parent validity bitmap; a row's validity is the validity of - /// its selected child. + /// A union has independent outer nullability in addition to the nullability of each variant. + /// This distinguishes a null union from a non-null union whose selected child is null. /// /// See [`UnionVariants`] for the type-tag conventions and accessors. - Union(UnionVariants), + Union(UnionVariants, Nullability), /// Dynamically typed values stored as Vortex scalars. /// @@ -152,7 +150,7 @@ impl PartialEq for DType { // StructFields handles its own Arc::ptr_eq in its PartialEq impl. (Self::Struct(a, na), Self::Struct(b, nb)) => na == nb && a == b, // UnionVariants handles its own Arc::ptr_eq in its PartialEq impl. - (Self::Union(a), Self::Union(b)) => a == b, + (Self::Union(a, na), Self::Union(b, nb)) => na == nb && a == b, (Self::Variant(a), Self::Variant(b)) => a == b, (Self::Extension(a), Self::Extension(b)) => a == b, // Every variant is listed in the first position so that adding a new diff --git a/vortex-array/src/dtype/serde/flatbuffers.rs b/vortex-array/src/dtype/serde/flatbuffers.rs index 6610ad5a580..a796490f58a 100644 --- a/vortex-array/src/dtype/serde/flatbuffers.rs +++ b/vortex-array/src/dtype/serde/flatbuffers.rs @@ -235,7 +235,7 @@ impl TryFrom for DType { .ok_or_else(|| vortex_err!("failed to parse union from flatbuffer"))?; let variants = UnionVariants::from_fb(fb_union, vfdt.buffer().clone(), vfdt.session.clone())?; - Ok(Self::Union(variants)) + Ok(Self::Union(variants, fb_union.nullable().into())) } fb::Type::Variant => { let fb_variant = fb @@ -381,7 +381,7 @@ impl WriteFlatBuffer for DType { ) .as_union_value() } - Self::Union(uv) => { + Self::Union(uv, n) => { let names = uv .names() .iter() @@ -411,6 +411,7 @@ impl WriteFlatBuffer for DType { names, dtypes, type_ids, + nullable: (*n).into(), }, ) .as_union_value() @@ -581,6 +582,7 @@ mod test { ], ) .unwrap(), + Nullability::NonNullable, ) } @@ -597,6 +599,7 @@ mod test { vec![DType::Null, DType::Utf8(Nullability::NonNullable)], ) .unwrap(), + Nullability::NonNullable, ); roundtrip_dtype(dtype); } @@ -614,6 +617,7 @@ mod test { vec![0, 5, u8::MAX], ) .unwrap(), + Nullability::Nullable, ); let bytes = dtype.write_flatbuffer_bytes().unwrap(); @@ -626,7 +630,7 @@ mod test { let deserialized = DType::try_from(view).unwrap(); assert_eq!(dtype, deserialized); - let DType::Union(uv) = &deserialized else { + let DType::Union(uv, _) = &deserialized else { panic!("Expected Union"); }; assert_eq!(uv.type_ids(), &[0, 5, u8::MAX]); @@ -649,6 +653,7 @@ mod test { vec![DType::Utf8(Nullability::NonNullable), struct_with_union], ) .unwrap(), + Nullability::Nullable, ); roundtrip_dtype(outer_union); @@ -704,6 +709,7 @@ mod test { names: Some(names), dtypes: Some(dtypes), type_ids: Some(type_ids), + nullable: false, }, ); diff --git a/vortex-array/src/dtype/serde/mod.rs b/vortex-array/src/dtype/serde/mod.rs index 17e07c244cb..c6837f4ab2d 100644 --- a/vortex-array/src/dtype/serde/mod.rs +++ b/vortex-array/src/dtype/serde/mod.rs @@ -187,12 +187,13 @@ mod test { fn test_serde_union_dtype_json_roundtrip() { let dtype = DType::Union( UnionVariants::new(["value"].into(), vec![DType::Utf8(Nullability::Nullable)]).unwrap(), + Nullability::Nullable, ); let json = serde_json::to_string(&dtype).unwrap(); assert_eq!( json, - r#"{"Union":{"names":["value"],"dtypes":[{"Utf8":true}],"type_ids":[0]}}"# + r#"{"Union":[{"names":["value"],"dtypes":[{"Utf8":true}],"type_ids":[0]},true]}"# ); let mut deserializer = serde_json::Deserializer::from_str(&json); let deserialized: DType = DTypeSerde::::new(&SESSION) diff --git a/vortex-array/src/dtype/serde/proto.rs b/vortex-array/src/dtype/serde/proto.rs index a0c27cb298b..8389a90200a 100644 --- a/vortex-array/src/dtype/serde/proto.rs +++ b/vortex-array/src/dtype/serde/proto.rs @@ -104,7 +104,7 @@ impl DType { }) .collect::>>()?; let variants = UnionVariants::try_new(names, dtypes, type_ids)?; - Ok(Self::Union(variants)) + Ok(Self::Union(variants, u.nullable.into())) } DtypeType::Variant(v) => Ok(Self::Variant(v.nullable.into())), DtypeType::Extension(e) => { @@ -173,13 +173,14 @@ impl TryFrom<&DType> for pb::DType { .collect::>>()?, nullable: (*null).into(), }), - DType::Union(uv) => DtypeType::Union(pb::Union { + DType::Union(uv, null) => DtypeType::Union(pb::Union { names: uv.names().iter().map(|n| n.as_ref().to_string()).collect(), dtypes: uv .variants() .map(|d| Self::try_from(&d)) .collect::>>()?, type_ids: uv.type_ids().iter().map(|t| *t as i32).collect(), + nullable: (*null).into(), }), DType::Variant(null) => DtypeType::Variant(pb::Variant { nullable: (*null).into(), @@ -416,22 +417,20 @@ mod tests { ], ) .unwrap(); - let dtype = DType::Union(variants); + let dtype = DType::Union(variants, Nullability::NonNullable); let converted = round_trip_dtype(&dtype); assert_eq!(dtype, converted); } #[test] - fn test_union_round_trip_proto_with_nullable_variant() { + fn test_union_round_trip_proto_with_inner_and_outer_nullability() { let variants = UnionVariants::new( ["null_variant", "str"].into(), vec![DType::Null, DType::Utf8(Nullability::NonNullable)], ) .unwrap(); - assert_eq!(variants.derived_nullability(), Nullability::Nullable); - - let dtype = DType::Union(variants); + let dtype = DType::Union(variants, Nullability::Nullable); let converted = round_trip_dtype(&dtype); assert_eq!(dtype, converted); } @@ -449,11 +448,11 @@ mod tests { ) .unwrap(); - let dtype = DType::Union(variants); + let dtype = DType::Union(variants, Nullability::Nullable); let converted = round_trip_dtype(&dtype); assert_eq!(dtype, converted); - let DType::Union(uv) = &converted else { + let DType::Union(uv, _) = &converted else { panic!("Expected Union"); }; assert_eq!(uv.type_ids(), &[0, 5, 7]); @@ -471,6 +470,7 @@ mod tests { ], // 256 does not fit in u8. type_ids: vec![256], + nullable: false, })), }; @@ -495,6 +495,7 @@ mod tests { ], ) .unwrap(), + Nullability::NonNullable, ); let struct_with_union = DType::Struct( @@ -511,6 +512,7 @@ mod tests { vec![DType::Utf8(Nullability::NonNullable), struct_with_union], ) .unwrap(), + Nullability::Nullable, ); let converted = round_trip_dtype(&outer_union); diff --git a/vortex-array/src/dtype/serde/serde.rs b/vortex-array/src/dtype/serde/serde.rs index 7f360382d7e..b3870b8c03e 100644 --- a/vortex-array/src/dtype/serde/serde.rs +++ b/vortex-array/src/dtype/serde/serde.rs @@ -116,7 +116,12 @@ impl Serialize for DType { state.serialize_field(n)?; state.end() } - DType::Union(uv) => serializer.serialize_newtype_variant("DType", 11, "Union", uv), + DType::Union(uv, n) => { + let mut state = serializer.serialize_tuple_variant("DType", 11, "Union", 2)?; + state.serialize_field(uv)?; + state.serialize_field(n)?; + state.end() + } DType::Variant(n) => serializer.serialize_newtype_variant("DType", 10, "Variant", n), DType::Extension(ext) => { serializer.serialize_newtype_variant("DType", 9, "Extension", ext) @@ -232,9 +237,9 @@ impl<'de> DeserializeSeed<'de> for DTypeSerde<'_, DType> { "Struct" => access.newtype_variant_seed(StructFieldsSeed { session: self.session, }), - "Union" => access - .newtype_variant_seed(DTypeSerde::::new(self.session)) - .map(DType::Union), + "Union" => access.newtype_variant_seed(UnionFieldsSeed { + session: self.session, + }), "Variant" => { let n = access.newtype_variant()?; Ok(DType::Variant(n)) @@ -405,6 +410,51 @@ impl<'de> DeserializeSeed<'de> for StructFieldsSeed<'_> { } } +struct UnionFieldsSeed<'a> { + session: &'a VortexSession, +} + +impl<'de> DeserializeSeed<'de> for UnionFieldsSeed<'_> { + type Value = DType; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct UnionVisitor<'a> { + session: &'a VortexSession, + } + + impl<'de> Visitor<'de> for UnionVisitor<'_> { + type Value = DType; + + fn expecting(&self, f: &mut Formatter) -> fmt::Result { + f.write_str("Union tuple (variants, nullability)") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let variants = seq + .next_element_seed(DTypeSerde::::new(self.session))? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + let nullability = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(1, &self))?; + Ok(DType::Union(variants, nullability)) + } + } + + deserializer.deserialize_tuple( + 2, + UnionVisitor { + session: self.session, + }, + ) + } +} + impl<'de> DeserializeSeed<'de> for DTypeSerde<'_, UnionVariants> { type Value = UnionVariants; diff --git a/vortex-array/src/dtype/union.rs b/vortex-array/src/dtype/union.rs index b14a3b84d21..7ce53155adc 100644 --- a/vortex-array/src/dtype/union.rs +++ b/vortex-array/src/dtype/union.rs @@ -14,7 +14,6 @@ use vortex_error::vortex_ensure_eq; use crate::dtype::DType; use crate::dtype::FieldDType; use crate::dtype::FieldNames; -use crate::dtype::Nullability; /// Type information for a union array. /// @@ -344,20 +343,10 @@ impl UnionVariants { pub fn child_index_to_tag(&self, index: usize) -> u8 { self.0.type_ids[index] } - - /// Returns the runtime-derived nullability of the union. - /// - /// This is not a zero-cost accessor: it scans every variant and materializes each - /// [`FieldDType`], which may be expensive when variants are still flatbuffer-backed views. - pub fn derived_nullability(&self) -> Nullability { - self.variants().any(|dtype| dtype.is_nullable()).into() - } } #[cfg(test)] mod tests { - use rstest::rstest; - use crate::dtype::DType; use crate::dtype::FieldNames; use crate::dtype::Nullability; @@ -476,39 +465,11 @@ mod tests { assert!(result.is_err()); } - #[rstest] - #[case::nullable_with_null_child( - vec![ - DType::Null, - DType::Primitive(PType::I32, Nullability::NonNullable), - ], - Nullability::Nullable, - )] - #[case::nullable_with_nullable_child( - vec![ - DType::Primitive(PType::I32, Nullability::NonNullable), - DType::Utf8(Nullability::Nullable), - ], - Nullability::Nullable, - )] - #[case::nonnullable( - vec![ - DType::Primitive(PType::I32, Nullability::NonNullable), - DType::Utf8(Nullability::NonNullable), - ], - Nullability::NonNullable, - )] - #[case::empty(vec![], Nullability::NonNullable)] - fn test_derived_nullability(#[case] dtypes: Vec, #[case] expected: Nullability) { - let names: Vec<&str> = (0..dtypes.len()).map(|i| ["a", "b", "c", "d"][i]).collect(); - let variants = UnionVariants::new(names.as_slice().into(), dtypes).unwrap(); - assert_eq!(variants.derived_nullability(), expected); - } - #[test] - fn test_nested_union_nullability() { + fn test_nested_union_nullability_is_independent() { let inner = DType::Union( UnionVariants::new(["value"].into(), vec![DType::Utf8(Nullability::Nullable)]).unwrap(), + Nullability::NonNullable, ); let outer = DType::Union( UnionVariants::new( @@ -519,28 +480,35 @@ mod tests { ], ) .unwrap(), + Nullability::Nullable, ); assert_eq!(outer.nullability(), Nullability::Nullable); + let variants = outer.as_union_variants_opt().unwrap(); + assert_eq!( + variants.variant_by_index(0).unwrap().nullability(), + Nullability::NonNullable + ); } #[test] - fn test_with_nullability_does_not_change_union_variants() { - let nonnullable = DType::Union(i32_variants()); - assert_eq!(nonnullable.as_nullable(), nonnullable); + fn test_with_nullability_changes_only_outer_union() { + let nonnullable = DType::Union(i32_variants(), Nullability::NonNullable); assert_eq!(nonnullable.nullability(), Nullability::NonNullable); - let nullable = DType::Union( - UnionVariants::new(["value"].into(), vec![DType::Utf8(Nullability::Nullable)]).unwrap(), - ); - assert_eq!(nullable.as_nonnullable(), nullable); + let nullable = nonnullable.as_nullable(); assert_eq!(nullable.nullability(), Nullability::Nullable); + assert_eq!( + nullable.as_union_variants_opt(), + nonnullable.as_union_variants_opt() + ); + assert_eq!(nullable.as_nonnullable(), nonnullable); } #[test] fn test_display() { let variants = i32_variants(); - let dtype = DType::Union(variants); + let dtype = DType::Union(variants, Nullability::NonNullable); assert_eq!(dtype.to_string(), "union(int=i32, str=utf8)"); let nullable = DType::Union( @@ -552,8 +520,13 @@ mod tests { ], ) .unwrap(), + Nullability::Nullable, ); assert_eq!(nullable.to_string(), "union(int=i32, maybe_str=utf8?)?"); + assert_eq!( + nullable.as_nonnullable().to_string(), + "union(int=i32, maybe_str=utf8?)" + ); } #[test] @@ -568,7 +541,7 @@ mod tests { vec![0, 5, 7], ) .unwrap(); - let dtype = DType::Union(variants); + let dtype = DType::Union(variants, Nullability::NonNullable); assert_eq!(dtype.to_string(), "union(a@0=i32, b@5=utf8, c@7=bool)"); } diff --git a/vortex-datafusion/src/convert/scalars.rs b/vortex-datafusion/src/convert/scalars.rs index e72a1f9ace3..ec262b479d9 100644 --- a/vortex-datafusion/src/convert/scalars.rs +++ b/vortex-datafusion/src/convert/scalars.rs @@ -832,7 +832,8 @@ mod tests { ["a"].into(), vec![DType::Primitive(PType::I32, Nullability::Nullable)], ) - .unwrap() + .unwrap(), + Nullability::Nullable, )))] fn unsupported_vortex_scalars_return_errors(#[case] scalar: Scalar) { let err = scalar.try_to_df().unwrap_err(); diff --git a/vortex-flatbuffers/flatbuffers/vortex-dtype/dtype.fbs b/vortex-flatbuffers/flatbuffers/vortex-dtype/dtype.fbs index f3b51c845b1..fb3b19513de 100644 --- a/vortex-flatbuffers/flatbuffers/vortex-dtype/dtype.fbs +++ b/vortex-flatbuffers/flatbuffers/vortex-dtype/dtype.fbs @@ -71,6 +71,7 @@ table Union { names: [string]; dtypes: [DType]; type_ids: [byte]; // length must equal dtypes.len(); interpreted as unsigned + nullable: bool; } union Type { diff --git a/vortex-flatbuffers/src/generated/dtype.rs b/vortex-flatbuffers/src/generated/dtype.rs index 953e826da82..56536ad0b94 100644 --- a/vortex-flatbuffers/src/generated/dtype.rs +++ b/vortex-flatbuffers/src/generated/dtype.rs @@ -1480,6 +1480,7 @@ impl<'a> Union<'a> { pub const VT_NAMES: ::flatbuffers::VOffsetT = 4; pub const VT_DTYPES: ::flatbuffers::VOffsetT = 6; pub const VT_TYPE_IDS: ::flatbuffers::VOffsetT = 8; + pub const VT_NULLABLE: ::flatbuffers::VOffsetT = 10; #[inline] pub unsafe fn init_from_table(table: ::flatbuffers::Table<'a>) -> Self { @@ -1494,6 +1495,7 @@ impl<'a> Union<'a> { if let Some(x) = args.type_ids { builder.add_type_ids(x); } if let Some(x) = args.dtypes { builder.add_dtypes(x); } if let Some(x) = args.names { builder.add_names(x); } + builder.add_nullable(args.nullable); builder.finish() } @@ -1519,6 +1521,13 @@ impl<'a> Union<'a> { // which contains a valid value in this slot unsafe { self._tab.get::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'a, i8>>>(Union::VT_TYPE_IDS, None)} } + #[inline] + pub fn nullable(&self) -> bool { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::(Union::VT_NULLABLE, Some(false)).unwrap()} + } } impl ::flatbuffers::Verifiable for Union<'_> { @@ -1530,6 +1539,7 @@ impl ::flatbuffers::Verifiable for Union<'_> { .visit_field::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'_, ::flatbuffers::ForwardsUOffset<&'_ str>>>>("names", Self::VT_NAMES, false)? .visit_field::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'_, ::flatbuffers::ForwardsUOffset>>>("dtypes", Self::VT_DTYPES, false)? .visit_field::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'_, i8>>>("type_ids", Self::VT_TYPE_IDS, false)? + .visit_field::("nullable", Self::VT_NULLABLE, false)? .finish(); Ok(()) } @@ -1538,6 +1548,7 @@ pub struct UnionArgs<'a> { pub names: Option<::flatbuffers::WIPOffset<::flatbuffers::Vector<'a, ::flatbuffers::ForwardsUOffset<&'a str>>>>, pub dtypes: Option<::flatbuffers::WIPOffset<::flatbuffers::Vector<'a, ::flatbuffers::ForwardsUOffset>>>>, pub type_ids: Option<::flatbuffers::WIPOffset<::flatbuffers::Vector<'a, i8>>>, + pub nullable: bool, } impl<'a> Default for UnionArgs<'a> { #[inline] @@ -1546,6 +1557,7 @@ impl<'a> Default for UnionArgs<'a> { names: None, dtypes: None, type_ids: None, + nullable: false, } } } @@ -1568,6 +1580,10 @@ impl<'a: 'b, 'b, A: ::flatbuffers::Allocator + 'a> UnionBuilder<'a, 'b, A> { self.fbb_.push_slot_always::<::flatbuffers::WIPOffset<_>>(Union::VT_TYPE_IDS, type_ids); } #[inline] + pub fn add_nullable(&mut self, nullable: bool) { + self.fbb_.push_slot::(Union::VT_NULLABLE, nullable, false); + } + #[inline] pub fn new(_fbb: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>) -> UnionBuilder<'a, 'b, A> { let start = _fbb.start_table(); UnionBuilder { @@ -1588,6 +1604,7 @@ impl ::core::fmt::Debug for Union<'_> { ds.field("names", &self.names()); ds.field("dtypes", &self.dtypes()); ds.field("type_ids", &self.type_ids()); + ds.field("nullable", &self.nullable()); ds.finish() } } diff --git a/vortex-flatbuffers/src/generated/message.rs b/vortex-flatbuffers/src/generated/message.rs index b3e4be76ef7..9cc48fe6be6 100644 --- a/vortex-flatbuffers/src/generated/message.rs +++ b/vortex-flatbuffers/src/generated/message.rs @@ -2,8 +2,8 @@ // @generated extern crate alloc; -use crate::array::*; use crate::dtype::*; +use crate::array::*; #[deprecated(since = "2.0.0", note = "Use associated constants instead. This will no longer be generated in 2021.")] pub const ENUM_MIN_MESSAGE_VERSION: u8 = 0; diff --git a/vortex-proto/proto/dtype.proto b/vortex-proto/proto/dtype.proto index 675a9156f22..11feef63a8c 100644 --- a/vortex-proto/proto/dtype.proto +++ b/vortex-proto/proto/dtype.proto @@ -78,6 +78,7 @@ message Union { repeated string names = 1; repeated DType dtypes = 2; repeated int32 type_ids = 3; // length must equal dtypes.len(); each value must fit in uint8 + bool nullable = 4; } message DType { diff --git a/vortex-proto/src/generated/vortex.dtype.rs b/vortex-proto/src/generated/vortex.dtype.rs index 984b4b6b71e..16188687ee5 100644 --- a/vortex-proto/src/generated/vortex.dtype.rs +++ b/vortex-proto/src/generated/vortex.dtype.rs @@ -80,6 +80,8 @@ pub struct Union { /// length must equal dtypes.len(); each value must fit in uint8 #[prost(int32, repeated, tag = "3")] pub type_ids: ::prost::alloc::vec::Vec, + #[prost(bool, tag = "4")] + pub nullable: bool, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct DType {