-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathstream.rs
More file actions
428 lines (390 loc) · 16.7 KB
/
stream.rs
File metadata and controls
428 lines (390 loc) · 16.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
//! Streaming translation between Serde data formats.
//!
//! xt's streaming transcoder is inspired by the [`serde_transcode`] crate advertised in the Serde
//! documentation, but has diverged significantly to support preserving original (de)serializer
//! error values, in contrast to `serde_transcode`'s approach of stringifying errors to meet Serde
//! API requirements.
//!
//! This was intended to facilitate a clean exit on [`BrokenPipe`](std::io::ErrorKind::BrokenPipe)
//! errors. However, experience shows that transcoding isn't the only hurdle for clean broken pipe
//! handling: the error types for some data formats don't expose I/O errors in their source chains,
//! making broken pipes difficult to discover from a coalesced `dyn Error` at the top of the call
//! stack. xt ultimately solved this problem with [`pipecheck`], moving the handling of broken
//! pipes directly into the `Write` path.
//!
//! The remaining benefit of preservation is to simplify error messages. Where `serde_transcode`
//! errors in the middle of collections tend to communicate the full "stack" of lines and columns
//! leading to a bad value, xt's errors point to the bad location alone.
//!
//! That said, `serde_transcode` is a simpler and more widely adopted implementation that supports
//! more of Serde than xt needs to. It should be your first choice.
//!
//! [`serde_transcode`]: https://github.com/sfackler/serde-transcode
use std::cell::Cell;
use std::error;
use std::fmt;
use serde::de::{self, DeserializeSeed, Deserializer};
use serde::ser::{self, Serialize, SerializeMap, SerializeSeq, Serializer};
/// The message used to generate generic serializer and deserializer errors.
///
/// When transcoding fails due to a serializer error, this message could make its way into the text
/// of the resulting deserializer error, depending on the specific deserializer in use.
const TRANSLATION_FAILED: &str = "translation failed";
/// Transcodes from a Serde `Deserializer` to a Serde `Serializer`.
///
/// The transcoding process forwards values produced by a deserializer directly to a serializer,
/// without collecting the entire output of the deserializer into an intermediate data structure.
///
/// An error in either the serializer or deserializer immediately halts transcoding.
/// See [`Error`] for information about how to interpret a transcoding error.
///
/// # Caveats
///
/// - The transcoder doesn't validate the deserializer's output in any meaningful way. For example,
/// it doesn't impose recursion limits on the deserialized data.
///
/// - Transcoding is only possible for self-describing data formats, where the types of values are
/// clear from the serialized representation itself.
///
/// - While the transcoding process doesn't inherently collect the deserializer's entire output,
/// certain (de)serializers may do so as part of their implementation.
///
/// - The current transcoder implementation doesn't handle all types supported by Serde. It only
/// handles types that a deserializer for a self-describing data format would reasonably be
/// expected to produce (i.e. not Rust-specific types like `Option<T>` or newtype structs).
pub(crate) fn transcode<'de, S, D>(ser: S, de: D) -> Result<S::Ok, Error<S::Error, D::Error>>
where
S: Serializer,
D: Deserializer<'de>,
{
let mut visitor = Visitor(Exchange::new(ser));
de.deserialize_any(&mut visitor).map_err(|de_err| {
let (source, ser_err) = visitor.0.into_error();
if matches!(source, ErrorSource::Ser) {
debug_assert!(
ser_err.is_some(),
"ErrorSource::Ser should include a serializer error"
);
}
if let (ErrorSource::Ser, Some(ser_err)) = (source, ser_err) {
Error::Ser(ser_err, de_err)
} else {
Error::De(de_err)
}
})
}
/// A [`transcode`] error.
///
/// The error indicates which side of the transcode originally triggered the failure, and provides
/// access to the original error value(s) generated by the serializer and/or deserializer.
#[derive(Debug)]
pub(crate) enum Error<S, D>
where
S: ser::Error,
D: de::Error,
{
/// The serializer triggered the transcode failure, for example due to an input value it
/// couldn't handle. The included deserializer error may provide useful context, such as the
/// line and column where transcoding failed.
Ser(S, D),
/// The deserializer triggered the transcode failure, for example due to a syntax error
/// in the input.
De(D),
}
impl<S, D> fmt::Display for Error<S, D>
where
S: ser::Error,
D: de::Error,
{
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::Ser(ser_err, de_err) => write!(f, "{de_err}: {ser_err}"),
Error::De(de_err) => fmt::Display::fmt(de_err, f),
}
}
}
impl<S, D> error::Error for Error<S, D>
where
S: ser::Error + 'static,
D: de::Error + 'static,
{
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match self {
Error::Ser(ser_err, _) => Some(ser_err),
Error::De(de_err) => Some(de_err),
}
}
}
/// The side of the transcode that failed first.
///
/// Since the transcoder may need to generate a synthetic serializer error to back out from a
/// deserializer failure, the fact that a serializer error made it up the call stack doesn't mean
/// the serializer caused the failure. The transcoder tracks this explicitly so it can discard
/// synthetic errors instead of returning them to the user.
#[derive(Clone, Copy)]
enum ErrorSource {
De,
Ser,
}
/// The state of a transcode step that can't cross normal Serde API boundaries.
///
/// Transcoding relies on special implementations of key Serde traits like [`Serialize`] and
/// [`de::Visitor`]. To preserve the original values of (de)serializer errors across Serde API
/// boundaries that can't represent them directly, and to account for the fact that many Serde
/// trait methods consume `self`, the transcoder's implementations of Serde trait methods follow a
/// common pattern:
///
/// 1. Take ownership of some "parent" value: either the serializer to which the next deserialized
/// value should be forwarded, or the deserializer that will produce the next value.
///
/// 2. Call some method of the parent with a value passed as an argument, consuming the parent and
/// producing a [`Result`] of generic value and error types.
///
/// 3. If the parent returned an error we can't return directly, stash that original error (and its
/// source) and instead return whatever error type our method signature requires, either by
/// extracting it from an `Exchange` or constructing it with a well-known message string (which
/// is the only capability that Serde's error traits require; they can't be constructed with
/// arbitrary payloads).
///
/// This type facilitates the pattern of exchanging parents and errors, generally as part of a
/// newtype struct that implements a specific Serde trait.
struct Exchange<P, E> {
// Only Forwarder actually needs interior mutability, due to `serialize` taking `&self`.
// However, two obvious ways of moving the interior mutability into Forwarder end up hurting
// transcoding performance:
//
// 1. When removing Cell from these fields but otherwise leaving the layout as-is, performance
// slows by as much as 15% - 25%, especially on formats like MessagePack that aren't
// expensive to encode or decode on their own.
//
// 2. When turning this into an enum (with empty, parent, and error states), the impact is
// closer to 5%. Not as bad, but still measurable and consistent.
//
// These numbers are based on Forwarder using Cell for interior mutability, but RefCell doesn't
// improve them either. I can only guess at the optimizations or CPU microarchitectural effects
// that might be at play. My suspicion is that this version somehow minimizes data copying, but
// I don't know how to confirm or deny that hypothesis.
//
// Serde traits are implemented on `&mut` references where possible to help mark where mutation
// is happening, even though `&` would technically work.
parent: Cell<Option<P>>,
error: Cell<Option<E>>,
source: Cell<ErrorSource>,
}
impl<P, E> Exchange<P, E> {
/// Creates a new exchange holding `parent`.
fn new(parent: P) -> Exchange<P, E> {
Exchange {
parent: Cell::new(Some(parent)),
error: Cell::new(None),
// Errors are assumed to come from the deserializer by default. Since the transcoder
// drives the serializer, it always knows when it needs to overwrite this.
source: Cell::new(ErrorSource::De),
}
}
/// Returns the contained parent.
///
/// # Panics
///
/// If the parent has already been taken.
fn take_parent(&self) -> P {
self.parent
.take()
.expect("Exchange should have its parent taken exactly once")
}
/// Stores an error for extraction by a parent, along with its source.
fn stash_error(&self, source: ErrorSource, error: E) {
self.source.set(source);
self.error.set(Some(error));
}
/// Returns the stashed error, if any, along with its source.
fn into_error(self) -> (ErrorSource, Option<E>) {
(self.source.get(), self.error.into_inner())
}
}
/// Takes the next value produced by a [`Deserializer`] and forwards it to a [`Serializer`].
///
/// The deserializer's [`Deserializer::deserialize_any`] drives the transcoding process by calling
/// the [`de::Visitor`] method corresponding to the type of value it sees in the input.
struct Visitor<S: Serializer>(Exchange<S, S::Error>);
impl<S: Serializer> Visitor<S> {
/// The boilerplate for visitor methods that merely forward scalars (booleans, numbers, etc.).
fn forward_scalar<F, E>(&mut self, serialize_with: F) -> Result<S::Ok, E>
where
F: FnOnce(S) -> Result<S::Ok, S::Error>,
E: de::Error,
{
let ser = self.0.take_parent();
serialize_with(ser).map_err(|ser_err| {
self.0.stash_error(ErrorSource::Ser, ser_err);
de::Error::custom(TRANSLATION_FAILED)
})
}
}
/// Implements the simplest [`de::Visitor`] methods that forward scalars to a [`Serializer`].
macro_rules! impl_forward_scalar_visitors {
( $( $name:ident($($arg:ident: $ty:ty)?) => $op:expr; )* ) => {
$(fn $name<E: de::Error>(self, $($arg: $ty)?) -> Result<Self::Value, E> {
self.forward_scalar($op)
})*
};
}
impl<'de, S: Serializer> de::Visitor<'de> for &mut Visitor<S> {
// It's important to note that we don't implement error preservation by having the visitor
// return `Result<S::Ok, S::Error>`, which might seem at first like the obvious way to do
// things. That approach requires one of two things to happen when the transcoder encounters a
// serializer error in the middle of a collection (sequence or map), neither of which is ideal:
//
// 1. The visitor can try to return the `Result` immediately, without having visited the entire
// collection. My experience is that some deserializers panic while unwinding this way.
// Maybe that's a bug in the deserializer; it's still not worth the risk.
//
// 2. To avoid the panics described in point 1, the visitor can "drain" the rest of the
// collection before it returns the final `Result`, e.g. by deserializing into `IgnoredAny`.
// This works, but wastes a lot of effort for an operation that we know will fail.
//
// Unlike the `Result` approach, the error stashing and mapping approach ensures that a failed
// transcode follows normal error handling paths in both the serializer and deserializer.
type Value = S::Ok;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any supported value")
}
impl_forward_scalar_visitors! {
visit_unit() => |ser| ser.serialize_unit();
visit_bool(v: bool) => |ser| ser.serialize_bool(v);
visit_i8(v: i8) => |ser| ser.serialize_i8(v);
visit_i16(v: i16) => |ser| ser.serialize_i16(v);
visit_i32(v: i32) => |ser| ser.serialize_i32(v);
visit_i64(v: i64) => |ser| ser.serialize_i64(v);
visit_i128(v: i128) => |ser| ser.serialize_i128(v);
visit_u8(v: u8) => |ser| ser.serialize_u8(v);
visit_u16(v: u16) => |ser| ser.serialize_u16(v);
visit_u32(v: u32) => |ser| ser.serialize_u32(v);
visit_u64(v: u64) => |ser| ser.serialize_u64(v);
visit_u128(v: u128) => |ser| ser.serialize_u128(v);
visit_f32(v: f32) => |ser| ser.serialize_f32(v);
visit_f64(v: f64) => |ser| ser.serialize_f64(v);
visit_char(v: char) => |ser| ser.serialize_char(v);
visit_str(v: &str) => |ser| ser.serialize_str(v);
visit_bytes(v: &[u8]) => |ser| ser.serialize_bytes(v);
}
fn visit_seq<A: de::SeqAccess<'de>>(self, mut de: A) -> Result<Self::Value, A::Error> {
let parent = self.0.take_parent();
let mut seq = parent.serialize_seq(de.size_hint()).map_err(|ser_err| {
self.0.stash_error(ErrorSource::Ser, ser_err);
de::Error::custom(TRANSLATION_FAILED)
})?;
loop {
let mut seed = SeqSeed(Exchange::new(&mut seq));
match de.next_element_seed(&mut seed) {
Ok(None) => break,
Ok(Some(())) => {}
Err(de_err) => {
if let (source, Some(ser_err)) = seed.0.into_error() {
self.0.stash_error(source, ser_err);
}
return Err(de_err);
}
}
}
seq.end().map_err(|ser_err| {
self.0.stash_error(ErrorSource::Ser, ser_err);
de::Error::custom(TRANSLATION_FAILED)
})
}
fn visit_map<A: de::MapAccess<'de>>(self, mut de: A) -> Result<Self::Value, A::Error> {
let parent = self.0.take_parent();
let mut map = parent.serialize_map(de.size_hint()).map_err(|ser_err| {
self.0.stash_error(ErrorSource::Ser, ser_err);
de::Error::custom(TRANSLATION_FAILED)
})?;
loop {
let mut key_seed = KeySeed(Exchange::new(&mut map));
match de.next_key_seed(&mut key_seed) {
Ok(None) => break,
Ok(Some(())) => {}
Err(de_err) => {
if let (source, Some(ser_err)) = key_seed.0.into_error() {
self.0.stash_error(source, ser_err);
}
return Err(de_err);
}
}
let mut value_seed = ValueSeed(Exchange::new(&mut map));
if let Err(de_err) = de.next_value_seed(&mut value_seed) {
if let (source, Some(ser_err)) = value_seed.0.into_error() {
self.0.stash_error(source, ser_err);
}
return Err(de_err);
}
}
map.end().map_err(|ser_err| {
self.0.stash_error(ErrorSource::Ser, ser_err);
de::Error::custom(TRANSLATION_FAILED)
})
}
}
/// Deserializable "values" that actually forward to a serializer.
///
/// To handle the elements of collections like sequences and maps, we need to hand the deserializer
/// a `Deserialize` or [`DeserializeSeed`] impl for each element. In return, it hands the impl a
/// [`Deserializer`] that produces the element.
///
/// Since our goal is to forward that element to a serializer, our impl is a [`DeserializeSeed`]
/// holding the serializer type for the kind of collection we're handling (sequence or map),
/// and calling the appropriate method on that type.
macro_rules! impl_seed_types {
( $( struct $name:ident ($trait:ident) => $x:expr; )* ) => {
$(struct $name<'ser, S: $trait>(Exchange<&'ser mut S, S::Error>);
impl<'de, 'ser, S: $trait> DeserializeSeed<'de> for &mut $name<'ser, S> {
type Value = ();
fn deserialize<D: Deserializer<'de>>(self, de: D) -> Result<(), D::Error> {
forward(de, &mut self.0, $x)
}
})*
};
}
impl_seed_types! {
struct SeqSeed(SerializeSeq) => |ser, elt| ser.serialize_element(elt);
struct KeySeed(SerializeMap) => |ser, key| ser.serialize_key(key);
struct ValueSeed(SerializeMap) => |ser, val| ser.serialize_value(val);
}
/// Forwards from a deserializer to the serializer held by a [seed](impl_seed_types).
fn forward<'de, D, S, SErr, F>(
de: D,
ser_exchange: &mut Exchange<S, SErr>,
serialize: F,
) -> Result<(), D::Error>
where
D: Deserializer<'de>,
F: FnOnce(S, &mut Forwarder<'de, D>) -> Result<(), SErr>,
{
let ser = ser_exchange.take_parent();
let mut forwarder = Forwarder(Exchange::new(de));
serialize(ser, &mut forwarder).map_err(|ser_err| {
let (source, de_err) = forwarder.0.into_error();
ser_exchange.stash_error(source, ser_err);
de_err.unwrap_or_else(|| de::Error::custom(TRANSLATION_FAILED))
})
}
/// A serializable "value" that actually serializes the next value from a [`Deserializer`].
///
/// The serializer held by a [seed](impl_seed_types) must be handed a [`Serialize`] impl for each
/// element. In return, the serializer hands the impl a [`Serializer`] that it can drive based on
/// the data it contains.
///
/// Since our goal is to take that value directly from a deserializer, our impl constructs a new
/// [`Visitor`] and begins a recursive round of the dance started by [`transcode`]. Unlike a normal
/// serializable value, a `Forwarder` panics if serialized more than once.
struct Forwarder<'de, D: Deserializer<'de>>(Exchange<D, D::Error>);
impl<'de, D: Deserializer<'de>> Serialize for Forwarder<'de, D> {
fn serialize<S: Serializer>(&self, ser: S) -> Result<S::Ok, S::Error> {
let de = self.0.take_parent();
let mut visitor = Visitor(Exchange::new(ser));
de.deserialize_any(&mut visitor).map_err(|de_err| {
let (source, ser_err) = visitor.0.into_error();
self.0.stash_error(source, de_err);
ser_err.unwrap_or_else(|| ser::Error::custom(TRANSLATION_FAILED))
})
}
}