Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions Lib/test/test_json/test_decode.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,28 @@ def test_limit_int(self):
with self.assertRaises(ValueError):
self.loads('1' * (maxdigits + 1))

def test_int_boundaries(self):
# Values around the signed/unsigned 64-bit limits and the
# 19-vs-20 digit fast-path threshold of the C accelerator.
for s in ['0', '-0',
'9223372036854775807', # LLONG_MAX
'9223372036854775808', # LLONG_MAX + 1
'-9223372036854775808', # LLONG_MIN
'-9223372036854775809', # LLONG_MIN - 1
'9999999999999999999', # largest 19-digit
'-9999999999999999999',
'18446744073709551615', # ULLONG_MAX (20 digits)
'18446744073709551616', # ULLONG_MAX + 1
'10000000000000000000', # smallest 20-digit
'-10000000000000000000']:
with self.subTest(s=s):
self.assertEqual(self.loads(s), int(s))

def test_long_float(self):
# A float longer than the C accelerator's stack buffer.
s = '0.' + '1' * 200
self.assertEqual(self.loads(s), float(s))


class TestPyDecode(TestDecode, PyTest): pass
class TestCDecode(TestDecode, CTest): pass
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Speed up :func:`json.loads` and :func:`json.load` parsing of numbers.
54 changes: 42 additions & 12 deletions Modules/_json.c
Original file line number Diff line number Diff line change
Expand Up @@ -997,7 +997,6 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_
Py_ssize_t idx = start;
int is_float = 0;
PyObject *rval;
PyObject *numstr = NULL;
PyObject *custom_func;

str = PyUnicode_DATA(pystr);
Expand Down Expand Up @@ -1064,32 +1063,63 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_

if (custom_func) {
/* copy the section we determined to be a number */
numstr = PyUnicode_FromKindAndData(kind,
(char*)str + kind * start,
idx - start);
PyObject *numstr = PyUnicode_FromKindAndData(kind,
(char*)str + kind * start,
idx - start);
if (numstr == NULL)
return NULL;
rval = PyObject_CallOneArg(custom_func, numstr);
Py_DECREF(numstr);
}
else {
Py_ssize_t i, n;
char *buf;

/* Fast path for integers with at most 19 digits (excluding the
optional minus sign): the magnitude always fits in an unsigned
long long, so construct the result from it directly and skip the
PyBytes allocation and the generic PyLong_FromString parser.
Integers with more digits fall back below. */
int neg = (PyUnicode_READ(kind, str, start) == '-');
if (!is_float && idx - start - neg <= 19) {
unsigned long long value = 0;
for (i = start + neg; i < idx; i++) {
value = value * 10 + (PyUnicode_READ(kind, str, i) - '0');
}
*next_idx_ptr = idx;
rval = PyLong_FromUnsignedLongLong(value);
if (neg && rval != NULL) {
Py_SETREF(rval, PyNumber_Negative(rval));
}
return rval;
}

/* Straight conversion to ASCII, to avoid costly conversion of
decimal unicode digits (which cannot appear here) */
n = idx - start;
numstr = PyBytes_FromStringAndSize(NULL, n);
if (numstr == NULL)
return NULL;
buf = PyBytes_AS_STRING(numstr);
char stackbuf[64];
PyObject *numstr = NULL;
char *buf;
if (n < (Py_ssize_t)sizeof(stackbuf)) {
buf = stackbuf;
buf[n] = '\0';
}
else {
numstr = PyBytes_FromStringAndSize(NULL, n);
if (numstr == NULL)
return NULL;
buf = PyBytes_AS_STRING(numstr);
}
for (i = 0; i < n; i++) {
buf[i] = (char) PyUnicode_READ(kind, str, i + start);
}
if (is_float)
rval = PyFloat_FromString(numstr);
if (is_float) {
double d = PyOS_string_to_double(buf, NULL, NULL);
rval = (d == -1.0 && PyErr_Occurred()) ? NULL : PyFloat_FromDouble(d);
}
else
rval = PyLong_FromString(buf, NULL, 10);
Py_XDECREF(numstr);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

numstr is only defined on lines 1068 and 1108.
You could declare it only in those places, and decref it in the same blocks.

}
Py_DECREF(numstr);
*next_idx_ptr = idx;
return rval;
}
Expand Down
Loading