bump jiter to 0.9, pyo3 to 0.24 (#1667)

Author: davidhewitt

Cargo.lock

@@ -29,7 +29,7 @@ rust-version = "1.75"
 [dependencies]
 # TODO it would be very nice to remove the "py-clone" feature as it can panic,
 # but needs a bit of work to make sure it's not used in the codebase
-pyo3 = { version = "0.23.5", features = ["generate-import-lib", "num-bigint", "py-clone"] }
+pyo3 = { version = "0.24", features = ["generate-import-lib", "num-bigint", "py-clone"] }
 regex = "1.11.1"
 strum = { version = "0.26.3", features = ["derive"] }
 strum_macros = "0.26.4"
@@ -45,7 +45,7 @@ idna = "1.0.3"
 base64 = "0.22.1"
 num-bigint = "0.4.6"
 uuid = "1.15.1"
-jiter = { version = "0.8.2", features = ["python"] }
+jiter = { version = "0.9.0", features = ["python"] }
 hex = "0.4.3"
 
 [lib]
@@ -73,12 +73,12 @@ debug = true
 strip = false
 
 [dev-dependencies]
-pyo3 = { version = "0.23.5", features = ["auto-initialize"] }
+pyo3 = { version = "0.24", features = ["auto-initialize"] }
 
 [build-dependencies]
 version_check = "0.9.5"
 # used where logic has to be version/distribution specific, e.g. pypy
-pyo3-build-config = { version = "0.23.5" }
+pyo3-build-config = { version = "0.24" }
 
 [lints.clippy]
 dbg_macro = "warn"

Cargo.toml

@@ -1,9 +1,8 @@
 use std::borrow::Cow;
 
-use jiter::{JsonArray, JsonObject, JsonValue, LazyIndexMap};
+use jiter::{JsonArray, JsonObject, JsonValue};
 use pyo3::prelude::*;
 use pyo3::types::{PyDict, PyList, PyString};
-use smallvec::SmallVec;
 use speedate::MicrosecondsPrecisionOverflowBehavior;
 use strum::EnumMessage;
 
@@ -62,7 +61,9 @@ impl<'py, 'data> Input<'py> for JsonValue<'data> {
         match self {
             JsonValue::Object(object) => {
                 let dict = PyDict::new(py);
-                for (k, v) in LazyIndexMap::iter(object) {
+                for (k, v) in object.as_slice() {
+                    // TODO: jiter doesn't deduplicate keys, so we should probably do that here to
+                    // avoid potential wasted work creating Python objects.
                     dict.set_item(k, v).unwrap();
                 }
                 Some(dict)
@@ -253,7 +254,14 @@ impl<'py, 'data> Input<'py> for JsonValue<'data> {
             JsonValue::Str(s) => Ok(string_to_vec(s).into()),
             JsonValue::Object(object) => {
                 // return keys iterator to match python's behavior
-                let keys: JsonArray = JsonArray::new(object.keys().map(|k| JsonValue::Str(k.clone())).collect());
+                // FIXME jiter doesn't deduplicate keys, should probably do that here before iteration.
+                let keys: JsonArray = JsonArray::new(
+                    object
+                        .as_slice()
+                        .iter()
+                        .map(|(k, _)| JsonValue::Str(k.clone()))
+                        .collect(),
+                );
                 Ok(GenericIterator::from(keys).into_static())
             }
             _ => Err(ValError::new(ErrorTypeDefaults::IterableType, self)),
@@ -543,19 +551,19 @@ impl<'data> ValidatedDict<'_> for &'_ JsonObject<'data> {
         &'a self,
         consumer: impl ConsumeIterator<ValResult<(Self::Key<'a>, Self::Item<'a>)>, Output = R>,
     ) -> ValResult<R> {
-        Ok(consumer.consume_iterator(LazyIndexMap::iter(self).map(|(k, v)| Ok((k.as_ref(), v)))))
+        Ok(consumer.consume_iterator(self.as_slice().iter().map(|(k, v)| Ok((k.as_ref(), v)))))
     }
 
     fn last_key(&self) -> Option<Self::Key<'_>> {
-        self.keys().last().map(AsRef::as_ref)
+        self.last().map(|(k, _)| k.as_ref())
     }
 }
 
 impl<'a, 'py, 'data> ValidatedList<'py> for &'a JsonArray<'data> {
     type Item = &'a JsonValue<'data>;
 
     fn len(&self) -> Option<usize> {
-        Some(SmallVec::len(self))
+        Some(Vec::len(self))
     }
     fn iterate<R>(self, consumer: impl ConsumeIterator<PyResult<Self::Item>, Output = R>) -> ValResult<R> {
         Ok(consumer.consume_iterator(self.iter().map(Ok)))
@@ -569,7 +577,7 @@ impl<'a, 'data> ValidatedTuple<'_> for &'a JsonArray<'data> {
     type Item = &'a JsonValue<'data>;
 
     fn len(&self) -> Option<usize> {
-        Some(SmallVec::len(self))
+        Some(Vec::len(self))
     }
     fn iterate<R>(self, consumer: impl ConsumeIterator<PyResult<Self::Item>, Output = R>) -> ValResult<R> {
         Ok(consumer.consume_iterator(self.iter().map(Ok)))
@@ -637,12 +645,12 @@ impl<'data> KeywordArgs<'_> for JsonObject<'data> {
         Self: 'a;
 
     fn len(&self) -> usize {
-        LazyIndexMap::len(self)
+        Vec::len(self)
     }
     fn get_item<'k>(&self, key: &'k LookupKey) -> ValResult<Option<(&'k LookupPath, Self::Item<'_>)>> {
         key.json_get(self)
     }
     fn iter(&self) -> impl Iterator<Item = ValResult<(Self::Key<'_>, Self::Item<'_>)>> {
-        LazyIndexMap::iter(self).map(|(k, v)| Ok((k.as_ref(), v)))
+        self.as_slice().iter().map(|(k, v)| Ok((k.as_ref(), v)))
     }
 }

src/input/input_json.rs

@@ -262,20 +262,33 @@ impl LookupKey {
         &'s self,
         dict: &'a JsonObject<'data>,
     ) -> ValResult<Option<(&'s LookupPath, &'a JsonValue<'data>)>> {
+        // FIXME: use of find_map in here probably leads to quadratic complexity
         match self {
-            Self::Simple(path) => match dict.get(path.first_key()) {
+            Self::Simple(path) => match dict
+                .iter()
+                .rev()
+                .find_map(|(k, v)| (k == path.first_key()).then_some(v))
+            {
                 Some(value) => {
                     debug_assert!(path.rest.is_empty());
                     Ok(Some((path, value)))
                 }
                 None => Ok(None),
             },
-            Self::Choice { path1, path2 } => match dict.get(path1.first_key()) {
+            Self::Choice { path1, path2 } => match dict
+                .iter()
+                .rev()
+                .find_map(|(k, v)| (k == path1.first_key()).then_some(v))
+            {
                 Some(value) => {
                     debug_assert!(path1.rest.is_empty());
                     Ok(Some((path1, value)))
                 }
-                None => match dict.get(path2.first_key()) {
+                None => match dict
+                    .iter()
+                    .rev()
+                    .find_map(|(k, v)| (k == path2.first_key()).then_some(v))
+                {
                     Some(value) => {
                         debug_assert!(path2.rest.is_empty());
                         Ok(Some((path2, value)))
@@ -287,7 +300,11 @@ impl LookupKey {
                 for path in path_choices {
                     // first step is different from the rest as we already know dict is JsonObject
                     // because of above checks, we know that path should have at least one element, hence unwrap
-                    let v: &JsonValue = match dict.get(path.first_item.key.as_str()) {
+                    let v: &JsonValue = match dict
+                        .iter()
+                        .rev()
+                        .find_map(|(k, v)| (k == path.first_key()).then_some(v))
+                    {
                         Some(v) => v,
                         None => continue,
                     };
@@ -527,7 +544,7 @@ impl PathItem {
 
     pub fn json_obj_get<'a, 'data>(&self, json_obj: &'a JsonObject<'data>) -> Option<&'a JsonValue<'data>> {
         match self {
-            Self::S(PathItemString { key, .. }) => json_obj.get(key.as_str()),
+            Self::S(PathItemString { key, .. }) => json_obj.iter().rev().find_map(|(k, v)| (k == key).then_some(v)),
             _ => None,
         }
     }

src/lookup_key.rs

@@ -668,7 +668,7 @@ impl DataclassValidator {
                 dc.call_method0(post_init)
             } else {
                 let args = post_init_kwargs.downcast::<PyTuple>()?;
-                dc.call_method1(post_init, args)
+                dc.call_method1(post_init, args.clone()) // FIXME should not need clone here
             };
             r.map_err(|e| convert_err(py, e, input))?;
         }