Skip to content

Commit 2bf9d9b

Browse files
samuelcolvinsamuelcolvin
committed
add "allow_partial" support
1 parent 9217019 commit 2bf9d9b

19 files changed

+197
-44
lines changed

‎python/pydantic_core/_pydantic_core.pyi

+16-1
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ class SchemaValidator:
9696
from_attributes: bool | None = None,
9797
context: Any | None = None,
9898
self_instance: Any | None = None,
99+
allow_partial: bool = False,
99100
) -> Any:
100101
"""
101102
Validate a Python object against the schema and return the validated object.
@@ -110,6 +111,8 @@ class SchemaValidator:
110111
[`info.context`][pydantic_core.core_schema.ValidationInfo.context].
111112
self_instance: An instance of a model set attributes on from validation, this is used when running
112113
validation from the `__init__` method of a model.
114+
allow_partial: Whether to allow partial validation, if `True` errors in the last element of sequences
115+
and mappings are ignored.
113116
114117
Raises:
115118
ValidationError: If validation fails.
@@ -143,6 +146,7 @@ class SchemaValidator:
143146
strict: bool | None = None,
144147
context: Any | None = None,
145148
self_instance: Any | None = None,
149+
allow_partial: bool = False,
146150
) -> Any:
147151
"""
148152
Validate JSON data directly against the schema and return the validated Python object.
@@ -160,6 +164,8 @@ class SchemaValidator:
160164
context: The context to use for validation, this is passed to functional validators as
161165
[`info.context`][pydantic_core.core_schema.ValidationInfo.context].
162166
self_instance: An instance of a model set attributes on from validation.
167+
allow_partial: Whether to allow partial validation, if `True` errors in the last element of sequences
168+
and mappings are ignored.
163169
164170
Raises:
165171
ValidationError: If validation fails or if the JSON data is invalid.
@@ -168,7 +174,14 @@ class SchemaValidator:
168174
Returns:
169175
The validated Python object.
170176
"""
171-
def validate_strings(self, input: _StringInput, *, strict: bool | None = None, context: Any | None = None) -> Any:
177+
def validate_strings(
178+
self,
179+
input: _StringInput,
180+
*,
181+
strict: bool | None = None,
182+
context: Any | None = None,
183+
allow_partial: bool = False,
184+
) -> Any:
172185
"""
173186
Validate a string against the schema and return the validated Python object.
174187
@@ -181,6 +194,8 @@ class SchemaValidator:
181194
If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used.
182195
context: The context to use for validation, this is passed to functional validators as
183196
[`info.context`][pydantic_core.core_schema.ValidationInfo.context].
197+
allow_partial: Whether to allow partial validation, if `True` errors in the last element of sequences
198+
and mappings are ignored.
184199
185200
Raises:
186201
ValidationError: If validation fails or if the JSON data is invalid.

‎python/pydantic_core/core_schema.py

+28-14
Original file line numberDiff line numberDiff line change
@@ -122,45 +122,58 @@ class CoreConfig(TypedDict, total=False):
122122

123123
class SerializationInfo(Protocol):
124124
@property
125-
def include(self) -> IncExCall: ...
125+
def include(self) -> IncExCall:
126+
...
126127

127128
@property
128-
def exclude(self) -> IncExCall: ...
129+
def exclude(self) -> IncExCall:
130+
...
129131

130132
@property
131133
def context(self) -> Any | None:
132134
"""Current serialization context."""
133135

134136
@property
135-
def mode(self) -> str: ...
137+
def mode(self) -> str:
138+
...
136139

137140
@property
138-
def by_alias(self) -> bool: ...
141+
def by_alias(self) -> bool:
142+
...
139143

140144
@property
141-
def exclude_unset(self) -> bool: ...
145+
def exclude_unset(self) -> bool:
146+
...
142147

143148
@property
144-
def exclude_defaults(self) -> bool: ...
149+
def exclude_defaults(self) -> bool:
150+
...
145151

146152
@property
147-
def exclude_none(self) -> bool: ...
153+
def exclude_none(self) -> bool:
154+
...
148155

149156
@property
150-
def serialize_as_any(self) -> bool: ...
157+
def serialize_as_any(self) -> bool:
158+
...
151159

152-
def round_trip(self) -> bool: ...
160+
def round_trip(self) -> bool:
161+
...
153162

154-
def mode_is_json(self) -> bool: ...
163+
def mode_is_json(self) -> bool:
164+
...
155165

156-
def __str__(self) -> str: ...
166+
def __str__(self) -> str:
167+
...
157168

158-
def __repr__(self) -> str: ...
169+
def __repr__(self) -> str:
170+
...
159171

160172

161173
class FieldSerializationInfo(SerializationInfo, Protocol):
162174
@property
163-
def field_name(self) -> str: ...
175+
def field_name(self) -> str:
176+
...
164177

165178

166179
class ValidationInfo(Protocol):
@@ -305,7 +318,8 @@ def plain_serializer_function_ser_schema(
305318

306319

307320
class SerializerFunctionWrapHandler(Protocol): # pragma: no cover
308-
def __call__(self, input_value: Any, index_key: int | str | None = None, /) -> Any: ...
321+
def __call__(self, input_value: Any, index_key: int | str | None = None, /) -> Any:
322+
...
309323

310324

311325
# (input_value: Any, serializer: SerializerFunctionWrapHandler, /) -> Any

‎src/errors/line_error.rs

+8
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,14 @@ impl ValLineError {
145145
self.error_type = error_type;
146146
self
147147
}
148+
149+
pub fn last_loc_item(&self) -> Option<&LocItem> {
150+
match &self.location {
151+
Location::Empty => None,
152+
// first because order is reversed
153+
Location::List(loc_items) => loc_items.first(),
154+
}
155+
}
148156
}
149157

150158
#[cfg_attr(debug_assertions, derive(Debug))]

‎src/errors/location.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use crate::lookup_key::{LookupPath, PathItem};
1212

1313
/// Used to store individual items of the error location, e.g. a string for key/field names
1414
/// or a number for array indices.
15-
#[derive(Clone)]
15+
#[derive(Clone, Eq, PartialEq)]
1616
#[cfg_attr(debug_assertions, derive(Debug))]
1717
pub enum LocItem {
1818
/// string type key, used to identify items from a dict or anything that implements `__getitem__`

‎src/errors/mod.rs

+38
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::validators::ValidationState;
12
use pyo3::prelude::*;
23

34
mod line_error;
@@ -30,3 +31,40 @@ pub fn py_err_string(py: Python, err: PyErr) -> String {
3031
Err(_) => "Unknown Error".to_string(),
3132
}
3233
}
34+
35+
/// If we're in `allow_partial` mode, whether all errors occurred in the last element of the input.
36+
pub fn sequence_valid_as_partial(state: &ValidationState, input_length: usize, errors: &[ValLineError]) -> bool {
37+
if !state.extra().allow_partial {
38+
return false;
39+
}
40+
// for the error to be in the last element, the index of all errors must be `input_length - 1`
41+
let last_index = (input_length - 1) as i64;
42+
errors.iter().all(|error| {
43+
if let Some(LocItem::I(loc_index)) = error.last_loc_item() {
44+
*loc_index == last_index
45+
} else {
46+
false
47+
}
48+
})
49+
}
50+
51+
/// If we're in `allow_partial` mode, whether all errors occurred in the last value of the input.
52+
pub fn mapping_valid_as_partial(
53+
state: &ValidationState,
54+
opt_last_key: Option<impl Into<LocItem>>,
55+
errors: &[ValLineError],
56+
) -> bool {
57+
if !state.extra().allow_partial {
58+
return false;
59+
}
60+
let Some(last_key) = opt_last_key.map(Into::into) else {
61+
return false;
62+
};
63+
errors.iter().all(|error| {
64+
if let Some(loc_item) = error.last_loc_item() {
65+
loc_item == &last_key
66+
} else {
67+
false
68+
}
69+
})
70+
}

‎src/input/input_abstract.rs

+5
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,8 @@ pub trait ValidatedDict<'py> {
236236
&'a self,
237237
consumer: impl ConsumeIterator<ValResult<(Self::Key<'a>, Self::Item<'a>)>, Output = R>,
238238
) -> ValResult<R>;
239+
// used in partial mode to check all errors occurred in the last key value pair
240+
fn last_key(&self) -> Option<Self::Key<'_>>;
239241
}
240242

241243
/// For validations from a list
@@ -276,6 +278,9 @@ impl<'py> ValidatedDict<'py> for Never {
276278
) -> ValResult<R> {
277279
unreachable!()
278280
}
281+
fn last_key(&self) -> Option<Self::Key<'_>> {
282+
unreachable!()
283+
}
279284
}
280285

281286
impl<'py> ValidatedList<'py> for Never {

‎src/input/input_json.rs

+4
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,10 @@ impl<'py, 'data> ValidatedDict<'py> for &'_ JsonObject<'data> {
516516
) -> ValResult<R> {
517517
Ok(consumer.consume_iterator(LazyIndexMap::iter(self).map(|(k, v)| Ok((k.as_ref(), v)))))
518518
}
519+
520+
fn last_key(&self) -> Option<Self::Key<'_>> {
521+
self.keys().last().map(AsRef::as_ref)
522+
}
519523
}
520524

521525
impl<'a, 'py, 'data> ValidatedList<'py> for &'a JsonArray<'data> {

‎src/input/input_python.rs

+8
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,14 @@ impl<'py> ValidatedDict<'py> for GenericPyMapping<'_, 'py> {
823823
Self::GetAttr(obj, _) => Ok(consumer.consume_iterator(iterate_attributes(obj)?)),
824824
}
825825
}
826+
827+
fn last_key(&self) -> Option<Self::Key<'_>> {
828+
match self {
829+
Self::Dict(dict) => dict.keys().iter().last(),
830+
Self::Mapping(mapping) => mapping.keys().ok()?.iter().ok()?.last()?.ok(),
831+
Self::GetAttr(_, _) => None,
832+
}
833+
}
826834
}
827835

828836
/// Container for all the collections (sized iterable containers) types, which

‎src/input/input_string.rs

+8
Original file line numberDiff line numberDiff line change
@@ -303,4 +303,12 @@ impl<'py> ValidatedDict<'py> for StringMappingDict<'py> {
303303
.map(|(key, val)| Ok((StringMapping::new_key(key)?, StringMapping::new_value(val)?))),
304304
))
305305
}
306+
307+
fn last_key(&self) -> Option<Self::Key<'_>> {
308+
self.0
309+
.keys()
310+
.iter()
311+
.last()
312+
.and_then(|key| StringMapping::new_key(key).ok())
313+
}
306314
}

‎src/input/return_enums.rs

+12-7
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ use pyo3::types::{PyBytes, PyComplex, PyFloat, PyFrozenSet, PyIterator, PyMappin
1717
use serde::{ser::Error, Serialize, Serializer};
1818

1919
use crate::errors::{
20-
py_err_string, ErrorType, ErrorTypeDefaults, InputValue, ToErrorValue, ValError, ValLineError, ValResult,
20+
py_err_string, sequence_valid_as_partial, ErrorType, ErrorTypeDefaults, InputValue, ToErrorValue, ValError,
21+
ValLineError, ValResult,
2122
};
2223
use crate::py_gc::PyGcTraverse;
2324
use crate::tools::{extract_i64, extract_int, new_py_string, py_err};
@@ -128,7 +129,9 @@ pub(crate) fn validate_iter_to_vec<'py>(
128129
) -> ValResult<Vec<PyObject>> {
129130
let mut output: Vec<PyObject> = Vec::with_capacity(capacity);
130131
let mut errors: Vec<ValLineError> = Vec::new();
131-
for (index, item_result) in iter.enumerate() {
132+
let mut index = 0;
133+
for item_result in iter {
134+
index += 1;
132135
let item = item_result.map_err(|e| any_next_error!(py, e, max_length_check.input, index))?;
133136
match validator.validate(py, item.borrow_input(), state) {
134137
Ok(item) => {
@@ -139,15 +142,15 @@ pub(crate) fn validate_iter_to_vec<'py>(
139142
max_length_check.incr()?;
140143
errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index)));
141144
if fail_fast {
142-
break;
145+
return Err(ValError::LineErrors(errors));
143146
}
144147
}
145148
Err(ValError::Omit) => (),
146149
Err(err) => return Err(err),
147150
}
148151
}
149152

150-
if errors.is_empty() {
153+
if errors.is_empty() || sequence_valid_as_partial(state, index, &errors) {
151154
Ok(output)
152155
} else {
153156
Err(ValError::LineErrors(errors))
@@ -197,7 +200,9 @@ pub(crate) fn validate_iter_to_set<'py>(
197200
fail_fast: bool,
198201
) -> ValResult<()> {
199202
let mut errors: Vec<ValLineError> = Vec::new();
200-
for (index, item_result) in iter.enumerate() {
203+
let mut index = 0;
204+
for item_result in iter {
205+
index += 1;
201206
let item = item_result.map_err(|e| any_next_error!(py, e, input, index))?;
202207
match validator.validate(py, item.borrow_input(), state) {
203208
Ok(item) => {
@@ -226,11 +231,11 @@ pub(crate) fn validate_iter_to_set<'py>(
226231
Err(err) => return Err(err),
227232
}
228233
if fail_fast && !errors.is_empty() {
229-
break;
234+
return Err(ValError::LineErrors(errors));
230235
}
231236
}
232237

233-
if errors.is_empty() {
238+
if errors.is_empty() || sequence_valid_as_partial(state, index, &errors) {
234239
Ok(())
235240
} else {
236241
Err(ValError::LineErrors(errors))

‎src/url.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ impl PyUrl {
4545
pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult<Self> {
4646
let schema_obj = SCHEMA_DEFINITION_URL
4747
.get_or_init(py, || build_schema_validator(py, "url"))
48-
.validate_python(py, url, None, None, None, None)?;
48+
.validate_python(py, url, None, None, None, None, false)?;
4949
schema_obj.extract(py)
5050
}
5151

@@ -225,7 +225,7 @@ impl PyMultiHostUrl {
225225
pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult<Self> {
226226
let schema_obj = SCHEMA_DEFINITION_MULTI_HOST_URL
227227
.get_or_init(py, || build_schema_validator(py, "multi-host-url"))
228-
.validate_python(py, url, None, None, None, None)?;
228+
.validate_python(py, url, None, None, None, None, false)?;
229229
schema_obj.extract(py)
230230
}
231231

‎src/validators/dict.rs

+4-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use pyo3::prelude::*;
33
use pyo3::types::PyDict;
44

55
use crate::build_tools::is_strict;
6-
use crate::errors::{LocItem, ValError, ValLineError, ValResult};
6+
use crate::errors::{sequence_valid_as_partial, LocItem, ValError, ValLineError, ValResult};
77
use crate::input::BorrowInput;
88
use crate::input::ConsumeIterator;
99
use crate::input::{Input, ValidatedDict};
@@ -109,8 +109,10 @@ where
109109
fn consume_iterator(self, iterator: impl Iterator<Item = ValResult<(Key, Value)>>) -> ValResult<PyObject> {
110110
let output = PyDict::new_bound(self.py);
111111
let mut errors: Vec<ValLineError> = Vec::new();
112+
let mut input_length = 0;
112113

113114
for item_result in iterator {
115+
input_length += 1;
114116
let (key, value) = item_result?;
115117
let output_key = match self.key_validator.validate(self.py, key.borrow_input(), self.state) {
116118
Ok(value) => Some(value),
@@ -140,7 +142,7 @@ where
140142
}
141143
}
142144

143-
if errors.is_empty() {
145+
if errors.is_empty() || sequence_valid_as_partial(self.state, input_length, &errors) {
144146
let input = self.input;
145147
length_check!(input, "Dictionary", self.min_length, self.max_length, output);
146148
Ok(output.into())

0 commit comments

Comments
 (0)