Add allow_partial (#1512)

Co-authored-by: David Hewitt <[email protected]>

Author: samuelcolvin

benches/main.rs

@@ -96,6 +96,7 @@ class SchemaValidator:
         from_attributes: bool | None = None,
         context: Any | None = None,
         self_instance: Any | None = None,
+        allow_partial: bool = False,
     ) -> Any:
         """
         Validate a Python object against the schema and return the validated object.
@@ -110,6 +111,8 @@ class SchemaValidator:
                 [`info.context`][pydantic_core.core_schema.ValidationInfo.context].
             self_instance: An instance of a model set attributes on from validation, this is used when running
                 validation from the `__init__` method of a model.
+            allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences
+                and mappings are ignored.
 
         Raises:
             ValidationError: If validation fails.
@@ -143,6 +146,7 @@ class SchemaValidator:
         strict: bool | None = None,
         context: Any | None = None,
         self_instance: Any | None = None,
+        allow_partial: bool = False,
     ) -> Any:
         """
         Validate JSON data directly against the schema and return the validated Python object.
@@ -160,6 +164,8 @@ class SchemaValidator:
             context: The context to use for validation, this is passed to functional validators as
                 [`info.context`][pydantic_core.core_schema.ValidationInfo.context].
             self_instance: An instance of a model set attributes on from validation.
+            allow_partial: Whether to allow partial validation; if `True` incomplete JSON will be parsed successfully
+                and errors in the last element of sequences and mappings are ignored.
 
         Raises:
             ValidationError: If validation fails or if the JSON data is invalid.
@@ -168,7 +174,14 @@ class SchemaValidator:
         Returns:
             The validated Python object.
         """
-    def validate_strings(self, input: _StringInput, *, strict: bool | None = None, context: Any | None = None) -> Any:
+    def validate_strings(
+        self,
+        input: _StringInput,
+        *,
+        strict: bool | None = None,
+        context: Any | None = None,
+        allow_partial: bool = False,
+    ) -> Any:
         """
         Validate a string against the schema and return the validated Python object.
 
@@ -181,6 +194,8 @@ class SchemaValidator:
                 If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used.
             context: The context to use for validation, this is passed to functional validators as
                 [`info.context`][pydantic_core.core_schema.ValidationInfo.context].
+            allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences
+                and mappings are ignored.
 
         Raises:
             ValidationError: If validation fails or if the JSON data is invalid.

python/pydantic_core/_pydantic_core.pyi

@@ -2840,7 +2840,7 @@ def typed_dict_field(
 
     Args:
         schema: The schema to use for the field
-        required: Whether the field is required
+        required: Whether the field is required, otherwise uses the value from `total` on the typed dict
         validation_alias: The alias(es) to use to find the field in the validation data
         serialization_alias: The alias to use as a key when serializing
         serialization_exclude: Whether to exclude the field when serializing
@@ -2916,7 +2916,7 @@ class MyTypedDict(TypedDict):
         ref: optional unique identifier of the schema, used to reference the schema in other places
         metadata: Any other information you want to include with the schema, not used by pydantic-core
         extra_behavior: The extra behavior to use for the typed dict
-        total: Whether the typed dict is total
+        total: Whether the typed dict is total, otherwise uses `typed_dict_total` from config
         populate_by_name: Whether the typed dict should populate by name
         serialization: Custom serialization schema
     """

python/pydantic_core/core_schema.py

@@ -145,6 +145,14 @@ impl ValLineError {
         self.error_type = error_type;
         self
     }
+
+    pub fn first_loc_item(&self) -> Option<&LocItem> {
+        match &self.location {
+            Location::Empty => None,
+            // last because order is reversed
+            Location::List(loc_items) => loc_items.last(),
+        }
+    }
 }
 
 #[cfg_attr(debug_assertions, derive(Debug))]

src/errors/line_error.rs

@@ -12,7 +12,7 @@ use crate::lookup_key::{LookupPath, PathItem};
 
 /// Used to store individual items of the error location, e.g. a string for key/field names
 /// or a number for array indices.
-#[derive(Clone)]
+#[derive(Clone, Eq, PartialEq)]
 #[cfg_attr(debug_assertions, derive(Debug))]
 pub enum LocItem {
     /// string type key, used to identify items from a dict or anything that implements `__getitem__`

src/errors/location.rs

@@ -236,6 +236,8 @@ pub trait ValidatedDict<'py> {
         &'a self,
         consumer: impl ConsumeIterator<ValResult<(Self::Key<'a>, Self::Item<'a>)>, Output = R>,
     ) -> ValResult<R>;
+    // used in partial mode to check all errors occurred in the last value
+    fn last_key(&self) -> Option<Self::Key<'_>>;
 }
 
 /// For validations from a list
@@ -276,6 +278,9 @@ impl<'py> ValidatedDict<'py> for Never {
     ) -> ValResult<R> {
         unreachable!()
     }
+    fn last_key(&self) -> Option<Self::Key<'_>> {
+        unreachable!()
+    }
 }
 
 impl<'py> ValidatedList<'py> for Never {

src/input/input_abstract.rs

@@ -516,6 +516,10 @@ impl<'py, 'data> ValidatedDict<'py> for &'_ JsonObject<'data> {
     ) -> ValResult<R> {
         Ok(consumer.consume_iterator(LazyIndexMap::iter(self).map(|(k, v)| Ok((k.as_ref(), v)))))
     }
+
+    fn last_key(&self) -> Option<Self::Key<'_>> {
+        self.keys().last().map(AsRef::as_ref)
+    }
 }
 
 impl<'a, 'py, 'data> ValidatedList<'py> for &'a JsonArray<'data> {

src/input/input_json.rs

@@ -823,6 +823,21 @@ impl<'py> ValidatedDict<'py> for GenericPyMapping<'_, 'py> {
             Self::GetAttr(obj, _) => Ok(consumer.consume_iterator(iterate_attributes(obj)?)),
         }
     }
+
+    fn last_key(&self) -> Option<Self::Key<'_>> {
+        match self {
+            Self::Dict(dict) => dict.keys().iter().last(),
+            // see https://github.com/pydantic/pydantic-core/pull/1512#discussion_r1826057970
+            Self::Mapping(mapping) => mapping
+                .call_method0(intern!(mapping.py(), "keys"))
+                .ok()?
+                .iter()
+                .ok()?
+                .last()?
+                .ok(),
+            Self::GetAttr(_, _) => None,
+        }
+    }
 }
 
 /// Container for all the collections (sized iterable containers) types, which

src/input/input_python.rs

@@ -303,4 +303,12 @@ impl<'py> ValidatedDict<'py> for StringMappingDict<'py> {
                 .map(|(key, val)| Ok((StringMapping::new_key(key)?, StringMapping::new_value(val)?))),
         ))
     }
+
+    fn last_key(&self) -> Option<Self::Key<'_>> {
+        self.0
+            .keys()
+            .iter()
+            .last()
+            .and_then(|key| StringMapping::new_key(key).ok())
+    }
 }

src/input/input_string.rs

@@ -128,7 +128,9 @@ pub(crate) fn validate_iter_to_vec<'py>(
 ) -> ValResult<Vec<PyObject>> {
     let mut output: Vec<PyObject> = Vec::with_capacity(capacity);
     let mut errors: Vec<ValLineError> = Vec::new();
-    for (index, item_result) in iter.enumerate() {
+
+    for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) {
+        state.allow_partial = is_last_partial;
         let item = item_result.map_err(|e| any_next_error!(py, e, max_length_check.input, index))?;
         match validator.validate(py, item.borrow_input(), state) {
             Ok(item) => {
@@ -137,9 +139,11 @@ pub(crate) fn validate_iter_to_vec<'py>(
             }
             Err(ValError::LineErrors(line_errors)) => {
                 max_length_check.incr()?;
-                errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
-                if fail_fast {
-                    break;
+                if !is_last_partial {
+                    errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
+                    if fail_fast {
+                        return Err(ValError::LineErrors(errors));
+                    }
                 }
             }
             Err(ValError::Omit) => (),
@@ -197,7 +201,9 @@ pub(crate) fn validate_iter_to_set<'py>(
     fail_fast: bool,
 ) -> ValResult<()> {
     let mut errors: Vec<ValLineError> = Vec::new();
-    for (index, item_result) in iter.enumerate() {
+
+    for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) {
+        state.allow_partial = is_last_partial;
         let item = item_result.map_err(|e| any_next_error!(py, e, input, index))?;
         match validator.validate(py, item.borrow_input(), state) {
             Ok(item) => {
@@ -220,13 +226,15 @@ pub(crate) fn validate_iter_to_set<'py>(
                 }
             }
             Err(ValError::LineErrors(line_errors)) => {
-                errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
+                if !is_last_partial {
+                    errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
+                }
             }
             Err(ValError::Omit) => (),
             Err(err) => return Err(err),
         }
         if fail_fast && !errors.is_empty() {
-            break;
+            return Err(ValError::LineErrors(errors));
         }
     }