Skip to content

Commit a1fa596

Browse files
samuelcolvindavidhewitt
samuelcolvin
and
davidhewitt
authored
Add allow_partial (#1512)
Co-authored-by: David Hewitt <[email protected]>
1 parent 5e95c05 commit a1fa596

27 files changed

+758
-100
lines changed

‎benches/main.rs

+219-51
Large diffs are not rendered by default.

‎python/pydantic_core/_pydantic_core.pyi

+16-1
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ class SchemaValidator:
9696
from_attributes: bool | None = None,
9797
context: Any | None = None,
9898
self_instance: Any | None = None,
99+
allow_partial: bool = False,
99100
) -> Any:
100101
"""
101102
Validate a Python object against the schema and return the validated object.
@@ -110,6 +111,8 @@ class SchemaValidator:
110111
[`info.context`][pydantic_core.core_schema.ValidationInfo.context].
111112
self_instance: An instance of a model set attributes on from validation, this is used when running
112113
validation from the `__init__` method of a model.
114+
allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences
115+
and mappings are ignored.
113116
114117
Raises:
115118
ValidationError: If validation fails.
@@ -143,6 +146,7 @@ class SchemaValidator:
143146
strict: bool | None = None,
144147
context: Any | None = None,
145148
self_instance: Any | None = None,
149+
allow_partial: bool = False,
146150
) -> Any:
147151
"""
148152
Validate JSON data directly against the schema and return the validated Python object.
@@ -160,6 +164,8 @@ class SchemaValidator:
160164
context: The context to use for validation, this is passed to functional validators as
161165
[`info.context`][pydantic_core.core_schema.ValidationInfo.context].
162166
self_instance: An instance of a model set attributes on from validation.
167+
allow_partial: Whether to allow partial validation; if `True` incomplete JSON will be parsed successfully
168+
and errors in the last element of sequences and mappings are ignored.
163169
164170
Raises:
165171
ValidationError: If validation fails or if the JSON data is invalid.
@@ -168,7 +174,14 @@ class SchemaValidator:
168174
Returns:
169175
The validated Python object.
170176
"""
171-
def validate_strings(self, input: _StringInput, *, strict: bool | None = None, context: Any | None = None) -> Any:
177+
def validate_strings(
178+
self,
179+
input: _StringInput,
180+
*,
181+
strict: bool | None = None,
182+
context: Any | None = None,
183+
allow_partial: bool = False,
184+
) -> Any:
172185
"""
173186
Validate a string against the schema and return the validated Python object.
174187
@@ -181,6 +194,8 @@ class SchemaValidator:
181194
If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used.
182195
context: The context to use for validation, this is passed to functional validators as
183196
[`info.context`][pydantic_core.core_schema.ValidationInfo.context].
197+
allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences
198+
and mappings are ignored.
184199
185200
Raises:
186201
ValidationError: If validation fails or if the JSON data is invalid.

‎python/pydantic_core/core_schema.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -2840,7 +2840,7 @@ def typed_dict_field(
28402840
28412841
Args:
28422842
schema: The schema to use for the field
2843-
required: Whether the field is required
2843+
required: Whether the field is required, otherwise uses the value from `total` on the typed dict
28442844
validation_alias: The alias(es) to use to find the field in the validation data
28452845
serialization_alias: The alias to use as a key when serializing
28462846
serialization_exclude: Whether to exclude the field when serializing
@@ -2916,7 +2916,7 @@ class MyTypedDict(TypedDict):
29162916
ref: optional unique identifier of the schema, used to reference the schema in other places
29172917
metadata: Any other information you want to include with the schema, not used by pydantic-core
29182918
extra_behavior: The extra behavior to use for the typed dict
2919-
total: Whether the typed dict is total
2919+
total: Whether the typed dict is total, otherwise uses `typed_dict_total` from config
29202920
populate_by_name: Whether the typed dict should populate by name
29212921
serialization: Custom serialization schema
29222922
"""

‎src/errors/line_error.rs

+8
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,14 @@ impl ValLineError {
145145
self.error_type = error_type;
146146
self
147147
}
148+
149+
pub fn first_loc_item(&self) -> Option<&LocItem> {
150+
match &self.location {
151+
Location::Empty => None,
152+
// last because order is reversed
153+
Location::List(loc_items) => loc_items.last(),
154+
}
155+
}
148156
}
149157

150158
#[cfg_attr(debug_assertions, derive(Debug))]

‎src/errors/location.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use crate::lookup_key::{LookupPath, PathItem};
1212

1313
/// Used to store individual items of the error location, e.g. a string for key/field names
1414
/// or a number for array indices.
15-
#[derive(Clone)]
15+
#[derive(Clone, Eq, PartialEq)]
1616
#[cfg_attr(debug_assertions, derive(Debug))]
1717
pub enum LocItem {
1818
/// string type key, used to identify items from a dict or anything that implements `__getitem__`

‎src/input/input_abstract.rs

+5
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,8 @@ pub trait ValidatedDict<'py> {
236236
&'a self,
237237
consumer: impl ConsumeIterator<ValResult<(Self::Key<'a>, Self::Item<'a>)>, Output = R>,
238238
) -> ValResult<R>;
239+
// used in partial mode to check all errors occurred in the last value
240+
fn last_key(&self) -> Option<Self::Key<'_>>;
239241
}
240242

241243
/// For validations from a list
@@ -276,6 +278,9 @@ impl<'py> ValidatedDict<'py> for Never {
276278
) -> ValResult<R> {
277279
unreachable!()
278280
}
281+
fn last_key(&self) -> Option<Self::Key<'_>> {
282+
unreachable!()
283+
}
279284
}
280285

281286
impl<'py> ValidatedList<'py> for Never {

‎src/input/input_json.rs

+4
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,10 @@ impl<'py, 'data> ValidatedDict<'py> for &'_ JsonObject<'data> {
516516
) -> ValResult<R> {
517517
Ok(consumer.consume_iterator(LazyIndexMap::iter(self).map(|(k, v)| Ok((k.as_ref(), v)))))
518518
}
519+
520+
fn last_key(&self) -> Option<Self::Key<'_>> {
521+
self.keys().last().map(AsRef::as_ref)
522+
}
519523
}
520524

521525
impl<'a, 'py, 'data> ValidatedList<'py> for &'a JsonArray<'data> {

‎src/input/input_python.rs

+15
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,21 @@ impl<'py> ValidatedDict<'py> for GenericPyMapping<'_, 'py> {
823823
Self::GetAttr(obj, _) => Ok(consumer.consume_iterator(iterate_attributes(obj)?)),
824824
}
825825
}
826+
827+
fn last_key(&self) -> Option<Self::Key<'_>> {
828+
match self {
829+
Self::Dict(dict) => dict.keys().iter().last(),
830+
// see https://github.com/pydantic/pydantic-core/pull/1512#discussion_r1826057970
831+
Self::Mapping(mapping) => mapping
832+
.call_method0(intern!(mapping.py(), "keys"))
833+
.ok()?
834+
.iter()
835+
.ok()?
836+
.last()?
837+
.ok(),
838+
Self::GetAttr(_, _) => None,
839+
}
840+
}
826841
}
827842

828843
/// Container for all the collections (sized iterable containers) types, which

‎src/input/input_string.rs

+8
Original file line numberDiff line numberDiff line change
@@ -303,4 +303,12 @@ impl<'py> ValidatedDict<'py> for StringMappingDict<'py> {
303303
.map(|(key, val)| Ok((StringMapping::new_key(key)?, StringMapping::new_value(val)?))),
304304
))
305305
}
306+
307+
fn last_key(&self) -> Option<Self::Key<'_>> {
308+
self.0
309+
.keys()
310+
.iter()
311+
.last()
312+
.and_then(|key| StringMapping::new_key(key).ok())
313+
}
306314
}

‎src/input/return_enums.rs

+15-7
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,9 @@ pub(crate) fn validate_iter_to_vec<'py>(
128128
) -> ValResult<Vec<PyObject>> {
129129
let mut output: Vec<PyObject> = Vec::with_capacity(capacity);
130130
let mut errors: Vec<ValLineError> = Vec::new();
131-
for (index, item_result) in iter.enumerate() {
131+
132+
for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) {
133+
state.allow_partial = is_last_partial;
132134
let item = item_result.map_err(|e| any_next_error!(py, e, max_length_check.input, index))?;
133135
match validator.validate(py, item.borrow_input(), state) {
134136
Ok(item) => {
@@ -137,9 +139,11 @@ pub(crate) fn validate_iter_to_vec<'py>(
137139
}
138140
Err(ValError::LineErrors(line_errors)) => {
139141
max_length_check.incr()?;
140-
errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
141-
if fail_fast {
142-
break;
142+
if !is_last_partial {
143+
errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
144+
if fail_fast {
145+
return Err(ValError::LineErrors(errors));
146+
}
143147
}
144148
}
145149
Err(ValError::Omit) => (),
@@ -197,7 +201,9 @@ pub(crate) fn validate_iter_to_set<'py>(
197201
fail_fast: bool,
198202
) -> ValResult<()> {
199203
let mut errors: Vec<ValLineError> = Vec::new();
200-
for (index, item_result) in iter.enumerate() {
204+
205+
for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) {
206+
state.allow_partial = is_last_partial;
201207
let item = item_result.map_err(|e| any_next_error!(py, e, input, index))?;
202208
match validator.validate(py, item.borrow_input(), state) {
203209
Ok(item) => {
@@ -220,13 +226,15 @@ pub(crate) fn validate_iter_to_set<'py>(
220226
}
221227
}
222228
Err(ValError::LineErrors(line_errors)) => {
223-
errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
229+
if !is_last_partial {
230+
errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
231+
}
224232
}
225233
Err(ValError::Omit) => (),
226234
Err(err) => return Err(err),
227235
}
228236
if fail_fast && !errors.is_empty() {
229-
break;
237+
return Err(ValError::LineErrors(errors));
230238
}
231239
}
232240

‎src/serializers/fields.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ impl SerField {
5353
}
5454
}
5555

56-
pub fn get_key_py<'py>(&'py self, py: Python<'py>, extra: &Extra) -> &Bound<'py, PyAny> {
56+
pub fn get_key_py<'py>(&self, py: Python<'py>, extra: &Extra) -> &Bound<'py, PyAny> {
5757
if extra.by_alias {
5858
if let Some(ref alias_py) = self.alias_py {
5959
return alias_py.bind(py);

‎src/url.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ impl PyUrl {
4545
pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult<Self> {
4646
let schema_obj = SCHEMA_DEFINITION_URL
4747
.get_or_init(py, || build_schema_validator(py, "url"))
48-
.validate_python(py, url, None, None, None, None)?;
48+
.validate_python(py, url, None, None, None, None, false)?;
4949
schema_obj.extract(py)
5050
}
5151

@@ -225,7 +225,7 @@ impl PyMultiHostUrl {
225225
pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult<Self> {
226226
let schema_obj = SCHEMA_DEFINITION_MULTI_HOST_URL
227227
.get_or_init(py, || build_schema_validator(py, "multi-host-url"))
228-
.validate_python(py, url, None, None, None, None)?;
228+
.validate_python(py, url, None, None, None, None, false)?;
229229
schema_obj.extract(py)
230230
}
231231

‎src/validators/arguments.rs

+3
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,9 @@ impl Validator for ArgumentsValidator {
188188
input: &(impl Input<'py> + ?Sized),
189189
state: &mut ValidationState<'_, 'py>,
190190
) -> ValResult<PyObject> {
191+
// this validator does not yet support partial validation, disable it to avoid incorrect results
192+
state.allow_partial = false;
193+
191194
let args = input.validate_args()?;
192195

193196
let mut output_args: Vec<PyObject> = Vec::with_capacity(self.positional_params_count);

‎src/validators/dataclass.rs

+3
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@ impl Validator for DataclassArgsValidator {
145145
input: &(impl Input<'py> + ?Sized),
146146
state: &mut ValidationState<'_, 'py>,
147147
) -> ValResult<PyObject> {
148+
// this validator does not yet support partial validation, disable it to avoid incorrect results
149+
state.allow_partial = false;
150+
148151
let args = input.validate_dataclass_args(&self.dataclass_name)?;
149152

150153
let output_dict = PyDict::new_bound(py);

‎src/validators/definitions.rs

+3
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ impl Validator for DefinitionRefValidator {
7575
input: &(impl Input<'py> + ?Sized),
7676
state: &mut ValidationState<'_, 'py>,
7777
) -> ValResult<PyObject> {
78+
// this validator does not yet support partial validation, disable it to avoid incorrect results
79+
state.allow_partial = false;
80+
7881
self.definition.read(|validator| {
7982
let validator = validator.unwrap();
8083
if let Some(id) = input.as_python().map(py_identity) {

‎src/validators/dict.rs

+9-7
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,8 @@ where
110110
let output = PyDict::new_bound(self.py);
111111
let mut errors: Vec<ValLineError> = Vec::new();
112112

113-
for item_result in iterator {
113+
for (_, is_last_partial, item_result) in self.state.enumerate_last_partial(iterator) {
114+
self.state.allow_partial = false;
114115
let (key, value) = item_result?;
115116
let output_key = match self.key_validator.validate(self.py, key.borrow_input(), self.state) {
116117
Ok(value) => Some(value),
@@ -124,19 +125,20 @@ where
124125
Err(ValError::Omit) => continue,
125126
Err(err) => return Err(err),
126127
};
128+
self.state.allow_partial = is_last_partial;
127129
let output_value = match self.value_validator.validate(self.py, value.borrow_input(), self.state) {
128-
Ok(value) => Some(value),
130+
Ok(value) => value,
129131
Err(ValError::LineErrors(line_errors)) => {
130-
for err in line_errors {
131-
errors.push(err.with_outer_location(key.clone()));
132+
if !is_last_partial {
133+
errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(key.clone())));
132134
}
133-
None
135+
continue;
134136
}
135137
Err(ValError::Omit) => continue,
136138
Err(err) => return Err(err),
137139
};
138-
if let (Some(key), Some(value)) = (output_key, output_value) {
139-
output.set_item(key, value)?;
140+
if let Some(key) = output_key {
141+
output.set_item(key, output_value)?;
140142
}
141143
}
142144

‎src/validators/generator.rs

+5-2
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ impl Validator for GeneratorValidator {
6666
input: &(impl Input<'py> + ?Sized),
6767
state: &mut ValidationState<'_, 'py>,
6868
) -> ValResult<PyObject> {
69+
// this validator does not yet support partial validation, disable it to avoid incorrect results
70+
state.allow_partial = false;
71+
6972
let iterator = input.validate_iter()?.into_static();
7073
let validator = self.item_validator.as_ref().map(|v| {
7174
InternalValidator::new(
@@ -279,7 +282,7 @@ impl InternalValidator {
279282
self_instance: self.self_instance.as_ref().map(|data| data.bind(py)),
280283
cache_str: self.cache_str,
281284
};
282-
let mut state = ValidationState::new(extra, &mut self.recursion_guard);
285+
let mut state = ValidationState::new(extra, &mut self.recursion_guard, false);
283286
state.exactness = self.exactness;
284287
let result = self
285288
.validator
@@ -314,7 +317,7 @@ impl InternalValidator {
314317
self_instance: self.self_instance.as_ref().map(|data| data.bind(py)),
315318
cache_str: self.cache_str,
316319
};
317-
let mut state = ValidationState::new(extra, &mut self.recursion_guard);
320+
let mut state = ValidationState::new(extra, &mut self.recursion_guard, false);
318321
state.exactness = self.exactness;
319322
let result = self.validator.validate(py, input, &mut state).map_err(|e| {
320323
ValidationError::from_val_error(

‎src/validators/json.rs

+8-4
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
1-
use jiter::FloatMode;
21
use pyo3::intern;
32
use pyo3::prelude::*;
43
use pyo3::types::PyDict;
54

6-
use jiter::{JsonValue, PartialMode, PythonParse};
5+
use jiter::{FloatMode, JsonValue, PartialMode, PythonParse};
76

87
use crate::errors::{ErrorType, ErrorTypeDefaults, ValError, ValLineError, ValResult};
98
use crate::input::{EitherBytes, Input, InputType, ValidationMatch};
@@ -60,7 +59,8 @@ impl Validator for JsonValidator {
6059
let json_bytes = json_either_bytes.as_slice();
6160
match self.validator {
6261
Some(ref validator) => {
63-
let json_value = JsonValue::parse(json_bytes, true).map_err(|e| map_json_err(input, e, json_bytes))?;
62+
let json_value = JsonValue::parse_with_config(json_bytes, true, state.allow_partial)
63+
.map_err(|e| map_json_err(input, e, json_bytes))?;
6464
let mut json_state = state.rebind_extra(|e| {
6565
e.input_type = InputType::Json;
6666
});
@@ -70,7 +70,11 @@ impl Validator for JsonValidator {
7070
let parse_builder = PythonParse {
7171
allow_inf_nan: true,
7272
cache_mode: state.cache_str(),
73-
partial_mode: PartialMode::Off,
73+
partial_mode: if state.allow_partial {
74+
PartialMode::TrailingStrings
75+
} else {
76+
PartialMode::Off
77+
},
7478
catch_duplicate_keys: false,
7579
float_mode: FloatMode::Float,
7680
};

0 commit comments

Comments
 (0)