Skip to content

bpo-34602: Quadruple stack size on macOS when compiling with UBSAN #27309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ambv merged 4 commits into from
Sep 3, 2021
Merged

bpo-34602: Quadruple stack size on macOS when compiling with UBSAN #27309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a92ce66
Quadruple stack size to be able to run full test suite with sanitizers
ambv Jul 23, 2021
188315e
Only quadruple stack size if --with-undefined-behavior-sanitizer was …
ambv Aug 27, 2021
5d734c3
add Blurb
ambv Aug 27, 2021
7044e8f
Respond to Ronald's review
ambv Aug 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
When building CPython on macOS with ``./configure
--with-undefined-behavior-sanitizer --with-pydebug``, the stack size is now
quadrupled to allow for the entire test suite to pass.
13 changes: 6 additions & 7 deletions Python/thread_pthread.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,17 @@
#define THREAD_STACK_SIZE 0 /* use default stack size */
#endif

/* The default stack size for new threads on OSX and BSD is small enough that
/* The default stack size for new threads on BSD is small enough that
* we'll get hard crashes instead of 'maximum recursion depth exceeded'
* exceptions.
*
* The default stack sizes below are the empirically determined minimal stack
* The default stack size below is the empirically determined minimal stack
* sizes where a simple recursive function doesn't cause a hard crash.
*
* For macOS the value of THREAD_STACK_SIZE is determined in configure.ac
* as it also depends on the other configure options like chosen sanitizer
* runtimes.
*/
#if defined(__APPLE__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0
#undef THREAD_STACK_SIZE
/* Note: This matches the value of -Wl,-stack_size in configure.ac */
#define THREAD_STACK_SIZE 0x1000000
#endif
#if defined(__FreeBSD__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0
#undef THREAD_STACK_SIZE
#define THREAD_STACK_SIZE 0x400000
Expand Down
147 changes: 81 additions & 66 deletions configure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -827,11 +827,11 @@ with_trace_refs
with_assertions
enable_optimizations
with_lto
with_hash_algorithm
with_tzpath
with_address_sanitizer
with_memory_sanitizer
with_undefined_behavior_sanitizer
with_hash_algorithm
with_tzpath
with_libs
with_system_expat
with_system_ffi
Expand Down Expand Up @@ -1548,12 +1548,6 @@ Optional Packages:
--with-lto=[full|thin|no|yes]
enable Link-Time-Optimization in any build (default
is no)
--with-hash-algorithm=[fnv|siphash24]
select hash algorithm for use in Python/pyhash.c
(default is SipHash24)
--with-tzpath=<list of absolute paths separated by pathsep>
Select the default time zone search path for zoneinfo.TZPATH

--with-address-sanitizer
enable AddressSanitizer memory error detector,
'asan' (default is no)
Expand All @@ -1562,6 +1556,12 @@ Optional Packages:
--with-undefined-behavior-sanitizer
enable UndefinedBehaviorSanitizer undefined
behaviour detector, 'ubsan' (default is no)
--with-hash-algorithm=[fnv|siphash24]
select hash algorithm for use in Python/pyhash.c
(default is SipHash24)
--with-tzpath=<list of absolute paths separated by pathsep>
Select the default time zone search path for zoneinfo.TZPATH

--with-libs='lib1 ...' link against additional libs (default is no)
--with-system-expat build pyexpat module using an installed expat
library, see Doc/library/pyexpat.rst (default is no)
Expand Down Expand Up @@ -9602,6 +9602,65 @@ $as_echo "no" >&6; }
;;
esac

{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5
$as_echo_n "checking for --with-address-sanitizer... " >&6; }

# Check whether --with-address_sanitizer was given.
if test "${with_address_sanitizer+set}" = set; then :
withval=$with_address_sanitizer;
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
$as_echo "$withval" >&6; }
BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
LDFLAGS="-fsanitize=address $LDFLAGS"
# ASan works by controlling memory allocation, our own malloc interferes.
with_pymalloc="no"

else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi


{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5
$as_echo_n "checking for --with-memory-sanitizer... " >&6; }

# Check whether --with-memory_sanitizer was given.
if test "${with_memory_sanitizer+set}" = set; then :
withval=$with_memory_sanitizer;
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
$as_echo "$withval" >&6; }
BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
# MSan works by controlling memory allocation, our own malloc interferes.
with_pymalloc="no"

else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi


{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5
$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; }

# Check whether --with-undefined_behavior_sanitizer was given.
if test "${with_undefined_behavior_sanitizer+set}" = set; then :
withval=$with_undefined_behavior_sanitizer;
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
$as_echo "$withval" >&6; }
BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
LDFLAGS="-fsanitize=undefined $LDFLAGS"
with_ubsan="yes"

else

{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
with_ubsan="no"

fi


# Set info about shared libraries.


Expand Down Expand Up @@ -9812,9 +9871,20 @@ then
# Issue #18075: the default maximum stack size (8MBytes) is too
# small for the default recursion limit. Increase the stack size
# to ensure that tests don't crash
# Note: This matches the value of THREAD_STACK_SIZE in
# thread_pthread.h
LINKFORSHARED="-Wl,-stack_size,1000000 $LINKFORSHARED"
stack_size="1000000" # 16 MB
if test "$with_ubsan" == "yes"
then
# Undefined behavior sanitizer requires an even deeper stack
stack_size="4000000" # 64 MB
fi

LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED"


cat >>confdefs.h <<_ACEOF
#define THREAD_STACK_SIZE 0x$stack_size
_ACEOF


if test "$enable_framework"
then
Expand Down Expand Up @@ -10410,61 +10480,6 @@ fi



{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5
$as_echo_n "checking for --with-address-sanitizer... " >&6; }

# Check whether --with-address_sanitizer was given.
if test "${with_address_sanitizer+set}" = set; then :
withval=$with_address_sanitizer;
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
$as_echo "$withval" >&6; }
BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
LDFLAGS="-fsanitize=address $LDFLAGS"
# ASan works by controlling memory allocation, our own malloc interferes.
with_pymalloc="no"

else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi


{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5
$as_echo_n "checking for --with-memory-sanitizer... " >&6; }

# Check whether --with-memory_sanitizer was given.
if test "${with_memory_sanitizer+set}" = set; then :
withval=$with_memory_sanitizer;
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
$as_echo "$withval" >&6; }
BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
# MSan works by controlling memory allocation, our own malloc interferes.
with_pymalloc="no"

else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi


{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5
$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; }

# Check whether --with-undefined_behavior_sanitizer was given.
if test "${with_undefined_behavior_sanitizer+set}" = set; then :
withval=$with_undefined_behavior_sanitizer;
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
$as_echo "$withval" >&6; }
BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
LDFLAGS="-fsanitize=undefined $LDFLAGS"

else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi


# Most SVR4 platforms (e.g. Solaris) need -lsocket and -lnsl.
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for t_open in -lnsl" >&5
$as_echo_n "checking for t_open in -lnsl... " >&6; }
Expand Down
93 changes: 53 additions & 40 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2595,6 +2595,47 @@ case $ac_sys_system/$ac_sys_release in
;;
esac

AC_MSG_CHECKING(for --with-address-sanitizer)
AC_ARG_WITH(address_sanitizer,
AS_HELP_STRING([--with-address-sanitizer],
[enable AddressSanitizer memory error detector, 'asan' (default is no)]),
[
AC_MSG_RESULT($withval)
BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
LDFLAGS="-fsanitize=address $LDFLAGS"
# ASan works by controlling memory allocation, our own malloc interferes.
with_pymalloc="no"
],
[AC_MSG_RESULT(no)])

AC_MSG_CHECKING(for --with-memory-sanitizer)
AC_ARG_WITH(memory_sanitizer,
AS_HELP_STRING([--with-memory-sanitizer],
[enable MemorySanitizer allocation error detector, 'msan' (default is no)]),
[
AC_MSG_RESULT($withval)
BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
# MSan works by controlling memory allocation, our own malloc interferes.
with_pymalloc="no"
],
[AC_MSG_RESULT(no)])

AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer)
AC_ARG_WITH(undefined_behavior_sanitizer,
AS_HELP_STRING([--with-undefined-behavior-sanitizer],
[enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]),
[
AC_MSG_RESULT($withval)
BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
LDFLAGS="-fsanitize=undefined $LDFLAGS"
with_ubsan="yes"
],
[
AC_MSG_RESULT(no)
with_ubsan="no"
])

# Set info about shared libraries.
AC_SUBST(SHLIB_SUFFIX)
AC_SUBST(LDSHARED)
Expand Down Expand Up @@ -2798,9 +2839,18 @@ then
# Issue #18075: the default maximum stack size (8MBytes) is too
# small for the default recursion limit. Increase the stack size
# to ensure that tests don't crash
# Note: This matches the value of THREAD_STACK_SIZE in
# thread_pthread.h
LINKFORSHARED="-Wl,-stack_size,1000000 $LINKFORSHARED"
stack_size="1000000" # 16 MB
if test "$with_ubsan" == "yes"
then
# Undefined behavior sanitizer requires an even deeper stack
stack_size="4000000" # 64 MB
fi

LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED"

AC_DEFINE_UNQUOTED(THREAD_STACK_SIZE,
0x$stack_size,
[Custom thread stack size depending on chosen sanitizer runtimes.])

if test "$enable_framework"
then
Expand Down Expand Up @@ -3044,43 +3094,6 @@ esac
AC_MSG_RESULT("$TZPATH")])
AC_SUBST(TZPATH)

AC_MSG_CHECKING(for --with-address-sanitizer)
AC_ARG_WITH(address_sanitizer,
AS_HELP_STRING([--with-address-sanitizer],
[enable AddressSanitizer memory error detector, 'asan' (default is no)]),
[
AC_MSG_RESULT($withval)
BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
LDFLAGS="-fsanitize=address $LDFLAGS"
# ASan works by controlling memory allocation, our own malloc interferes.
with_pymalloc="no"
],
[AC_MSG_RESULT(no)])

AC_MSG_CHECKING(for --with-memory-sanitizer)
AC_ARG_WITH(memory_sanitizer,
AS_HELP_STRING([--with-memory-sanitizer],
[enable MemorySanitizer allocation error detector, 'msan' (default is no)]),
[
AC_MSG_RESULT($withval)
BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
# MSan works by controlling memory allocation, our own malloc interferes.
with_pymalloc="no"
],
[AC_MSG_RESULT(no)])

AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer)
AC_ARG_WITH(undefined_behavior_sanitizer,
AS_HELP_STRING([--with-undefined-behavior-sanitizer],
[enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]),
[
AC_MSG_RESULT($withval)
BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
LDFLAGS="-fsanitize=undefined $LDFLAGS"
],
[AC_MSG_RESULT(no)])

# Most SVR4 platforms (e.g. Solaris) need -lsocket and -lnsl.
AC_CHECK_LIB(nsl, t_open, [LIBS="-lnsl $LIBS"]) # SVR4
AC_CHECK_LIB(socket, socket, [LIBS="-lsocket $LIBS"], [], $LIBS) # SVR4 sockets
Expand Down
3 changes: 3 additions & 0 deletions pyconfig.h.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1509,6 +1509,9 @@
(which you can't on SCO ODT 3.0). */
#undef SYS_SELECT_WITH_SYS_TIME

/* Custom thread stack size depending on chosen sanitizer runtimes. */
#undef THREAD_STACK_SIZE

/* Library needed by timemodule.c: librt may be needed for clock_gettime() */
#undef TIMEMODULE_LIB

Expand Down