Skip to content

Support unicode array type. #2896

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 1 commit into from
Aug 21, 2021
Merged

Support unicode array type. #2896

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 2 additions & 79 deletions Lib/test/test_array.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import array
# from array import _array_reconstructor as array_reconstructor # XXX: RUSTPYTHON

# sizeof_wchar = array.array('u').itemsize # XXX: RUSTPYTHON
sizeof_wchar = array.array('u').itemsize


class ArraySubclass(array.array):
Expand All @@ -25,10 +25,7 @@ class ArraySubclassWithKwargs(array.array):
def __init__(self, typecode, newarg=None):
array.array.__init__(self)

# TODO: RUSTPYTHON
# We did not support typecode u for unicode yet
# typecodes = 'ubBhHiIlLfdqQ'
typecodes = 'bBhHiIlLfdqQ'
typecodes = 'ubBhHiIlLfdqQ'

class MiscTest(unittest.TestCase):

Expand Down Expand Up @@ -1091,8 +1088,6 @@ def test_sizeof_without_buffer(self):
basesize = support.calcvobjsize('Pn2Pi')
support.check_sizeof(self, a, basesize)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_initialize_with_unicode(self):
if self.typecode != 'u':
with self.assertRaises(TypeError) as cm:
Expand Down Expand Up @@ -1121,8 +1116,6 @@ def test_free_after_iterating(self):

class StringTest(BaseTest):

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_setitem(self):
super().test_setitem()
a = array.array(self.typecode, self.example)
Expand All @@ -1136,173 +1129,105 @@ class UnicodeTest(StringTest, unittest.TestCase):
outside = str('\x33')
minitemsize = 2

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_add(self):
super().test_add()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_buffer(self):
super().test_buffer()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_buffer_info(self):
super().test_buffer_info()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_byteswap(self):
super().test_byteswap()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_cmp(self):
super().test_cmp()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_constructor(self):
super().test_constructor()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_constructor_with_iterable_argument(self):
super().test_constructor_with_iterable_argument()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_copy(self):
super().test_copy()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_count(self):
super().test_count()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_coveritertraverse(self):
super().test_coveritertraverse()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_deepcopy(self):
super().test_deepcopy()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_delitem(self):
super().test_delitem()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_exhausted_iterator(self):
super().test_exhausted_iterator()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_extend(self):
super().test_extend()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_extended_getslice(self):
super().test_extended_getslice()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_extended_set_del_slice(self):
super().test_extended_set_del_slice()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_fromarray(self):
super().test_fromarray()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_getitem(self):
super().test_getitem()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_getslice(self):
super().test_getslice()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_iadd(self):
super().test_iadd()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_imul(self):
super().test_imul()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_index(self):
super().test_index()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_insert(self):
super().test_insert()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_len(self):
super().test_len()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_mul(self):
super().test_mul()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_pop(self):
super().test_pop()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_remove(self):
super().test_remove()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_repr(self):
super().test_repr()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_reverse(self):
super().test_reverse()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_setslice(self):
super().test_setslice()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_str(self):
super().test_str()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_tofrombytes(self):
super().test_tofrombytes()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_tofromlist(self):
super().test_tofromlist()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_unicode(self):
self.assertRaises(TypeError, array.array, 'b', 'foo')

Expand All @@ -1323,8 +1248,6 @@ def test_unicode(self):

self.assertRaises(TypeError, a.fromunicode)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_issue17223(self):
# this used to crash
if sizeof_wchar == 4:
Expand Down
32 changes: 20 additions & 12 deletions Lib/test/test_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2596,8 +2596,6 @@ def setUp(self):
def tearDown(self):
support.unlink(support.TESTFN)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_constructor(self):
r = self.BytesIO(b"\xc3\xa9\n\n")
b = self.BufferedReader(r, 1000)
Expand Down Expand Up @@ -2935,8 +2933,6 @@ def test_error_through_destructor(self):

# Systematic tests of the text I/O API

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_basic_io(self):
for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65):
for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le":
Expand Down Expand Up @@ -2988,8 +2984,6 @@ def multi_line_test(self, f, enc):
rlines.append((pos, line))
self.assertEqual(rlines, wlines)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_telling(self):
f = self.open(support.TESTFN, "w+", encoding="utf-8")
p0 = f.tell()
Expand Down Expand Up @@ -3608,8 +3602,6 @@ def seekable(self): return True
F.tell = lambda x: 0
t = self.TextIOWrapper(F(), encoding='utf-8')

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_reconfigure_encoding_read(self):
# latin1 -> utf8
# (latin1 can decode utf-8 encoded string)
Expand Down Expand Up @@ -3762,6 +3754,26 @@ class CTextIOWrapperTest(TextIOWrapperTest):
io = io
shutdown_error = "RuntimeError: could not find io module state"

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_constructor(self):
super().test_constructor()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_reconfigure_encoding_read(self):
super().test_reconfigure_encoding_read()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_basic_io(self):
super().test_basic_io()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_telling(self):
super().test_telling()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_uninitialized(self):
Expand Down Expand Up @@ -3917,8 +3929,6 @@ def test_newlines(self):
def test_line_buffering(self):
super().test_line_buffering()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_seeking_too(self):
super().test_seeking_too()

Expand All @@ -3927,8 +3937,6 @@ def test_seeking_too(self):
def test_bufio_write_through(self):
super().test_bufio_write_through()

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_seeking(self):
super().test_seeking()

Expand Down
6 changes: 0 additions & 6 deletions Lib/test/test_ntpath.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,6 @@ def test_splitext(self):
tester('ntpath.splitext("xx\\foo.bar.ext")', ('xx\\foo.bar', '.ext'))
tester('ntpath.splitext("c:a/b\\c.d")', ('c:a/b\\c', '.d'))

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_splitdrive(self):
tester('ntpath.splitdrive("c:\\foo\\bar")',
('c:', '\\foo\\bar'))
Expand Down Expand Up @@ -479,8 +477,6 @@ def test_expandvars(self):
tester('ntpath.expandvars("\'%foo%\'%bar")', "\'%foo%\'%bar")
tester('ntpath.expandvars("bar\'%foo%")', "bar\'%foo%")

# TODO: RUSTPYTHON
@unittest.expectedFailure
@unittest.skipUnless(support.FS_NONASCII, 'need support.FS_NONASCII')
def test_expandvars_nonascii(self):
def check(value, expected):
Expand Down Expand Up @@ -730,8 +726,6 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
pathmodule = ntpath
attributes = ['relpath']

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_expandvars_nonascii(self):
super().test_expandvars_nonascii()

Expand Down
2 changes: 0 additions & 2 deletions Lib/test/test_re.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1398,8 +1398,6 @@ def test_bug_6561(self):
for x in not_decimal_digits:
self.assertIsNone(re.match(r'^\d$', x))

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_empty_array(self):
# SF buf 1647541
import array
Expand Down
1 change: 1 addition & 0 deletions Lib/test/test_strtod.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ def test_short_halfway_cases(self):
digits *= 5
exponent -= 1

# TODO: RUSTPYTHON fails on debug mode
def test_halfway_cases(self):
# test halfway cases for the round-half-to-even rule
for i in range(100 * TEST_SIZE):
Expand Down
2 changes: 0 additions & 2 deletions Lib/test/test_traceback.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ def syntax_error_with_caret_non_ascii(self):
def syntax_error_bad_indentation2(self):
compile(" print(2)", "?", "exec")

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_caret(self):
err = self.get_exception_format(self.syntax_error_with_caret,
SyntaxError)
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_unicode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,8 +199,6 @@ def test_count(self):
self.checkequal(0, 'a' * 10, 'count', 'a\U00100304')
self.checkequal(0, '\u0102' * 10, 'count', '\u0102\U00100304')

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_find(self):
string_tests.CommonTest.test_find(self)
# test implementation details of the memchr fast path
Expand Down Expand Up @@ -260,8 +258,6 @@ def test_rfind(self):
self.checkequal(-1, 'a' * 100, 'rfind', '\U00100304a')
self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304\u0102')

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_index(self):
string_tests.CommonTest.test_index(self)
self.checkequalnofix(0, 'abcdefghiabc', 'index', '')
Expand Down
3 changes: 1 addition & 2 deletions vm/src/builtins/memory.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ type PyMemoryViewRef = PyRef<PyMemoryView>;
#[pyimpl(with(Hashable, Comparable, AsBuffer))]
impl PyMemoryView {
fn parse_format(format: &str, vm: &VirtualMachine) -> PyResult<FormatSpec> {
FormatSpec::parse(format)
.map_err(|msg| vm.new_exception_msg(vm.ctx.types.memoryview_type.clone(), msg))
FormatSpec::parse(format, vm)
}

pub fn from_buffer(
Expand Down
Loading