Skip to content

bpo-36285: Array module int overflow #12317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
serhiy-storchaka merged 10 commits into from
Mar 20, 2019
Merged

bpo-36285: Array module int overflow #12317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7236e62
Add tests for large arrays
sth Mar 10, 2019
5a93ffe
Guard tests @bigmemtest
sth Mar 11, 2019
74999ad
Use Py_ssize_t in loops
sth Mar 11, 2019
1fada13
Clean up example data creation
sth Mar 12, 2019
9cab158
Test fewer count() and index() cases because they are slow
sth Mar 12, 2019
5eabf42
Improve test name
sth Mar 13, 2019
0cc6265
Explain memuse
sth Mar 13, 2019
c337e90
Add test for slices
sth Mar 13, 2019
255e5fd
Add news
sth Mar 14, 2019
506f7f9
Constructing the base array can take more than memuse=1
sth Mar 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 141 additions & 0 deletions Lib/test/test_array.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import unittest
from test import support
from test.support import _2G
import weakref
import pickle
import operator
Expand Down Expand Up @@ -1396,5 +1397,145 @@ def test_alloc_overflow(self):
self.fail("Array of size > maxsize created - MemoryError expected")


class LargeArrayTest(unittest.TestCase):
typecode = 'b'

def example(self, size):
# We assess a base memuse of <=2.125 for constructing this array
base = array.array(self.typecode, [0, 1, 2, 3, 4, 5, 6, 7]) * (size // 8)
base += array.array(self.typecode, [99]*(size % 8) + [8, 9, 10, 11])
return base

@support.bigmemtest(_2G, memuse=2.125)
def test_example_data(self, size):
example = self.example(size)
self.assertEqual(len(example), size+4)

@support.bigmemtest(_2G, memuse=2.125)
def test_access(self, size):
example = self.example(size)
self.assertEqual(example[0], 0)
self.assertEqual(example[-(size+4)], 0)
self.assertEqual(example[size], 8)
self.assertEqual(example[-4], 8)
self.assertEqual(example[size+3], 11)
self.assertEqual(example[-1], 11)

@support.bigmemtest(_2G, memuse=2.125+1)
def test_slice(self, size):
example = self.example(size)
self.assertEqual(list(example[:4]), [0, 1, 2, 3])
self.assertEqual(list(example[-4:]), [8, 9, 10, 11])
part = example[1:-1]
self.assertEqual(len(part), size+2)
self.assertEqual(part[0], 1)
self.assertEqual(part[-1], 10)
del part
part = example[::2]
self.assertEqual(len(part), (size+5)//2)
self.assertEqual(list(part[:4]), [0, 2, 4, 6])
if size % 2:
self.assertEqual(list(part[-2:]), [9, 11])
else:
self.assertEqual(list(part[-2:]), [8, 10])

@support.bigmemtest(_2G, memuse=2.125)
def test_count(self, size):
example = self.example(size)
self.assertEqual(example.count(0), size//8)
self.assertEqual(example.count(11), 1)

@support.bigmemtest(_2G, memuse=2.125)
def test_append(self, size):
example = self.example(size)
example.append(12)
self.assertEqual(example[-1], 12)

@support.bigmemtest(_2G, memuse=2.125)
def test_extend(self, size):
example = self.example(size)
example.extend(iter([12, 13, 14, 15]))
self.assertEqual(len(example), size+8)
self.assertEqual(list(example[-8:]), [8, 9, 10, 11, 12, 13, 14, 15])

@support.bigmemtest(_2G, memuse=2.125)
def test_frombytes(self, size):
example = self.example(size)
example.frombytes(b'abcd')
self.assertEqual(len(example), size+8)
self.assertEqual(list(example[-8:]), [8, 9, 10, 11] + list(b'abcd'))

@support.bigmemtest(_2G, memuse=2.125)
def test_fromlist(self, size):
example = self.example(size)
example.fromlist([12, 13, 14, 15])
self.assertEqual(len(example), size+8)
self.assertEqual(list(example[-8:]), [8, 9, 10, 11, 12, 13, 14, 15])

@support.bigmemtest(_2G, memuse=2.125)
def test_index(self, size):
example = self.example(size)
self.assertEqual(example.index(0), 0)
self.assertEqual(example.index(1), 1)
self.assertEqual(example.index(7), 7)
self.assertEqual(example.index(11), size+3)

@support.bigmemtest(_2G, memuse=2.125)
def test_insert(self, size):
example = self.example(size)
example.insert(0, 12)
example.insert(10, 13)
example.insert(size+1, 14)
self.assertEqual(len(example), size+7)
self.assertEqual(example[0], 12)
self.assertEqual(example[10], 13)
self.assertEqual(example[size+1], 14)

@support.bigmemtest(_2G, memuse=2.125)
def test_pop(self, size):
example = self.example(size)
self.assertEqual(example.pop(0), 0)
self.assertEqual(example[0], 1)
self.assertEqual(example.pop(size+1), 10)
self.assertEqual(example[size+1], 11)
self.assertEqual(example.pop(1), 2)
self.assertEqual(example[1], 3)
self.assertEqual(len(example), size+1)
self.assertEqual(example.pop(), 11)
self.assertEqual(len(example), size)

@support.bigmemtest(_2G, memuse=2.125)
def test_remove(self, size):
example = self.example(size)
example.remove(0)
self.assertEqual(len(example), size+3)
self.assertEqual(example[0], 1)
example.remove(10)
self.assertEqual(len(example), size+2)
self.assertEqual(example[size], 9)
self.assertEqual(example[size+1], 11)

@support.bigmemtest(_2G, memuse=2.125)
def test_reverse(self, size):
example = self.example(size)
example.reverse()
self.assertEqual(len(example), size+4)
self.assertEqual(example[0], 11)
self.assertEqual(example[3], 8)
self.assertEqual(example[-1], 0)
example.reverse()
self.assertEqual(len(example), size+4)
self.assertEqual(list(example[:4]), [0, 1, 2, 3])
self.assertEqual(list(example[-4:]), [8, 9, 10, 11])

# list takes about 9 bytes per element
@support.bigmemtest(_2G, memuse=2.125+9)
def test_tolist(self, size):
example = self.example(size)
ls = example.tolist()
self.assertEqual(len(ls), len(example))
self.assertEqual(ls[:8], list(example[:8]))
self.assertEqual(ls[-8:], list(example[-8:]))

if __name__ == "__main__":
unittest.main()
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Library/2019-03-14-01-09-59.bpo-36285.G-usj8.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix integer overflows in the array module. Patch by Stephan Hohe.
8 changes: 4 additions & 4 deletions Modules/arraymodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1174,7 +1174,7 @@ static PyObject *
array_array_remove(arrayobject *self, PyObject *v)
/*[clinic end generated code: output=bef06be9fdf9dceb input=0b1e5aed25590027]*/
{
int i;
Py_ssize_t i;

for (i = 0; i < Py_SIZE(self); i++) {
PyObject *selfi;
Expand Down Expand Up @@ -2029,7 +2029,7 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype,
switch (mformat_code) {
case IEEE_754_FLOAT_LE:
case IEEE_754_FLOAT_BE: {
int i;
Py_ssize_t i;
int le = (mformat_code == IEEE_754_FLOAT_LE) ? 1 : 0;
Py_ssize_t itemcount = Py_SIZE(items) / 4;
const unsigned char *memstr =
Expand All @@ -2051,7 +2051,7 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype,
}
case IEEE_754_DOUBLE_LE:
case IEEE_754_DOUBLE_BE: {
int i;
Py_ssize_t i;
int le = (mformat_code == IEEE_754_DOUBLE_LE) ? 1 : 0;
Py_ssize_t itemcount = Py_SIZE(items) / 8;
const unsigned char *memstr =
Expand Down Expand Up @@ -2106,7 +2106,7 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype,
case UNSIGNED_INT64_BE:
case SIGNED_INT64_LE:
case SIGNED_INT64_BE: {
int i;
Py_ssize_t i;
const struct mformatdescr mf_descr =
mformat_descriptors[mformat_code];
Py_ssize_t itemcount = Py_SIZE(items) / mf_descr.size;
Expand Down