Skip to content

Commit f6dd539

Browse files
committed
Fix unicode array pickling
1 parent d4ac911 commit f6dd539

2 files changed

Lines changed: 54 additions & 37 deletions

File tree

‎extra_tests/snippets/stdlib_array.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from testutils import assert_raises
22
from array import array
3+
from pickle import dumps, loads
34

45
a1 = array("b", [0, 1, 2, 3])
56

@@ -96,4 +97,10 @@ def test_array_frombytes():
9697
with assert_raises(IndexError):
9798
a[0] = 42
9899
with assert_raises(IndexError):
99-
del a[42]
100+
del a[42]
101+
102+
test_str = '🌉abc🌐def🌉🌐'
103+
u = array('u', test_str)
104+
assert u.__reduce_ex__(1)[1][1] == list(test_str)
105+
assert str(loads(dumps(u, 1))) == f"array('u', '{test_str}')"
106+
assert loads(dumps(u, 1)) == loads(dumps(u, 3))

‎vm/src/stdlib/array.rs‎

Lines changed: 46 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -568,12 +568,8 @@ mod array {
568568
fn byteswap(self) -> Self {
569569
Self(self.0.swap_bytes())
570570
}
571-
fn to_object(self, vm: &VirtualMachine) -> PyObjectRef {
572-
vm.ctx.new_str(
573-
char::from_u32(self.0 as u32)
574-
.unwrap_or_default()
575-
.to_string(),
576-
)
571+
fn to_object(self, _vm: &VirtualMachine) -> PyObjectRef {
572+
unreachable!()
577573
}
578574
}
579575

@@ -767,6 +763,38 @@ mod array {
767763
}
768764
}
769765

766+
fn _wchar_bytes_to_string(
767+
bytes: &[u8],
768+
item_size: usize,
769+
vm: &VirtualMachine,
770+
) -> PyResult<String> {
771+
if item_size == 2 {
772+
// safe because every configuration of bytes for the types we support are valid
773+
let utf16 = unsafe {
774+
std::slice::from_raw_parts(
775+
bytes.as_ptr() as *const u16,
776+
bytes.len() / std::mem::size_of::<u16>(),
777+
)
778+
};
779+
Ok(String::from_utf16_lossy(utf16))
780+
} else {
781+
// safe because every configuration of bytes for the types we support are valid
782+
let chars = unsafe {
783+
std::slice::from_raw_parts(
784+
bytes.as_ptr() as *const u32,
785+
bytes.len() / std::mem::size_of::<u32>(),
786+
)
787+
};
788+
chars
789+
.iter()
790+
.map(|&ch| {
791+
// cpython issue 17223
792+
u32_to_char(ch).map_err(|msg| vm.new_value_error(msg))
793+
})
794+
.try_collect()
795+
}
796+
}
797+
770798
fn _unicode_to_wchar_bytes(utf8: &str, item_size: usize) -> Vec<u8> {
771799
if item_size == 2 {
772800
utf8.encode_utf16()
@@ -807,31 +835,7 @@ mod array {
807835
));
808836
}
809837
let bytes = array.get_bytes();
810-
if self.itemsize() == 2 {
811-
// safe because every configuration of bytes for the types we support are valid
812-
let utf16 = unsafe {
813-
std::slice::from_raw_parts(
814-
bytes.as_ptr() as *const u16,
815-
bytes.len() / std::mem::size_of::<u16>(),
816-
)
817-
};
818-
Ok(String::from_utf16_lossy(utf16))
819-
} else {
820-
// safe because every configuration of bytes for the types we support are valid
821-
let chars = unsafe {
822-
std::slice::from_raw_parts(
823-
bytes.as_ptr() as *const u32,
824-
bytes.len() / std::mem::size_of::<u32>(),
825-
)
826-
};
827-
chars
828-
.iter()
829-
.map(|&ch| {
830-
// cpython issue 17223
831-
u32_to_char(ch).map_err(|msg| vm.new_value_error(msg))
832-
})
833-
.try_collect()
834-
}
838+
Self::_wchar_bytes_to_string(bytes, self.itemsize(), vm)
835839
}
836840

837841
#[pymethod]
@@ -1075,7 +1079,7 @@ mod array {
10751079
vm: &VirtualMachine,
10761080
) -> PyResult<(PyObjectRef, PyTupleRef, Option<PyDictRef>)> {
10771081
if proto < 3 {
1078-
return Ok(Self::reduce(zelf, vm));
1082+
return Self::reduce(zelf, vm);
10791083
}
10801084
let array = zelf.read();
10811085
let cls = zelf.as_object().clone_class().into_object();
@@ -1096,16 +1100,22 @@ mod array {
10961100
fn reduce(
10971101
zelf: PyRef<Self>,
10981102
vm: &VirtualMachine,
1099-
) -> (PyObjectRef, PyTupleRef, Option<PyDictRef>) {
1103+
) -> PyResult<(PyObjectRef, PyTupleRef, Option<PyDictRef>)> {
11001104
let array = zelf.read();
11011105
let cls = zelf.as_object().clone_class().into_object();
11021106
let typecode = vm.ctx.new_str(array.typecode_str());
1103-
let values = vm.ctx.new_list(array.get_objects(vm));
1104-
(
1107+
let values = if array.typecode() == 'u' {
1108+
let s = Self::_wchar_bytes_to_string(array.get_bytes(), array.itemsize(), vm)?;
1109+
s.chars().map(|x| x.into_pyobject(vm)).collect()
1110+
} else {
1111+
array.get_objects(vm)
1112+
};
1113+
let values = vm.ctx.new_list(values);
1114+
Ok((
11051115
cls,
11061116
PyTupleRef::with_elements(vec![typecode, values], &vm.ctx),
11071117
zelf.as_object().dict(),
1108-
)
1118+
))
11091119
}
11101120
}
11111121

0 commit comments

Comments
 (0)