ARROW-538: [C++] Set up AddressSanitizer (ASAN) builds

xhochy · wesm · commit 0bdfd5efb2d7 · 2017-02-08T15:07:39.000-05:00
Most of the infrastructure was already in place, only needed to fix the gtest build. We will now build with AddressSanitizer activated on OSX. Author: Uwe L. Korn <uwelk@xhochy.com> Closes apache#324 from xhochy/ARROW-538 and squashes the following commits: c2f8dda [Uwe L. Korn] Don't run AddressSanitizer on Travis f6b65e5 [Uwe L. Korn] Explicitly detected 3.6 8a20d91 [Uwe L. Korn] Log detected COMPILER_VERSION in error message acf3f69 [Uwe L. Korn] ARROW-538: [C++] Set up AddressSanitizer (ASAN) builds
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -428,10 +428,12 @@ if(ARROW_BUILD_TESTS)
 
   if("$ENV{GTEST_HOME}" STREQUAL "")
     if(APPLE)
-      set(GTEST_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
+      set(GTEST_CMAKE_CXX_FLAGS "-fPIC -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
     else()
       set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
     endif()
+    string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
+    set(GTEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}")
 
     set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
     set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh
@@ -82,8 +82,8 @@ function setup_sanitizers() {
 
   # Enable leak detection even under LLVM 3.4, where it was disabled by default.
   # This flag only takes effect when running an ASAN build.
-  ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
-  export ASAN_OPTIONS
+  # ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
+  # export ASAN_OPTIONS
 
   # Set up suppressions for LeakSanitizer
   LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt"
diff --git a/cpp/build-support/sanitize-blacklist.txt b/cpp/build-support/sanitize-blacklist.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Workaround for a problem with gmock where a runtime error is caused by a call on a null pointer,
+# on a mocked object.
+# Seen error:
+# thirdparty/gmock-1.7.0/include/gmock/gmock-spec-builders.h:1529:12: runtime error: member call on null pointer of type 'testing::internal::ActionResultHolder<void>'
+fun:*testing*internal*InvokeWith*
diff --git a/cpp/cmake_modules/CompilerInfo.cmake b/cpp/cmake_modules/CompilerInfo.cmake
@@ -30,6 +30,12 @@ elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
   set(COMPILER_FAMILY "clang")
   string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
     COMPILER_VERSION "${COMPILER_VERSION_FULL}")
+
+# LLVM 3.6 on Mac OS X 10.9 and later
+elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM 3\\.6\\..*")
+  set(COMPILER_FAMILY "clang")
+  set(COMPILER_VERSION "3.6.0svn")
+
 # clang on Mac OS X 10.9 and later
 elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
   set(COMPILER_FAMILY "clang")
diff --git a/cpp/cmake_modules/san-config.cmake b/cpp/cmake_modules/san-config.cmake
@@ -94,8 +94,9 @@ if ("${ARROW_USE_UBSAN}" OR "${ARROW_USE_ASAN}" OR "${ARROW_USE_TSAN}")
     # Require clang 3.4 or newer; clang 3.3 has issues with TSAN and pthread
     # symbol interception.
     if("${COMPILER_VERSION}" VERSION_LESS "3.4")
-      message(SEND_ERROR "Must use clang 3.4 or newer to run a sanitizer build."
-        " Try using clang from $NATIVE_TOOLCHAIN/")
+        message(SEND_ERROR "Must use clang 3.4 or newer to run a sanitizer build."
+        " Detected unsupported version ${COMPILER_VERSION}."
+        " Try using clang from $NATIVE_TOOLCHAIN/.")
     endif()
     add_definitions("-fsanitize-blacklist=${BUILD_SUPPORT_DIR}/sanitize-blacklist.txt")
   else()
diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
@@ -67,11 +67,14 @@ TEST_F(TestBuffer, Resize) {
 }
 
 TEST_F(TestBuffer, ResizeOOM) {
+  // This test doesn't play nice with AddressSanitizer
+#ifndef ADDRESS_SANITIZER
   // realloc fails, even though there may be no explicit limit
   PoolBuffer buf;
   ASSERT_OK(buf.Resize(100));
   int64_t to_alloc = std::numeric_limits<int64_t>::max();
   ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));
+#endif
 }
 
 TEST_F(TestBuffer, EqualsWithSameContent) {
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
@@ -32,7 +32,9 @@ TEST_F(TestDefaultMemoryPool, MemoryTracking) {
 }
 
 TEST_F(TestDefaultMemoryPool, OOM) {
+#ifndef ADDRESS_SANITIZER
   this->TestOOM();
+#endif
 }
 
 TEST_F(TestDefaultMemoryPool, Reallocate) {
@@ -41,7 +43,7 @@ TEST_F(TestDefaultMemoryPool, Reallocate) {
 
 // Death tests and valgrind are known to not play well 100% of the time. See
 // googletest documentation
-#ifndef ARROW_VALGRIND
+#if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER))
 
 TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
   MemoryPool* pool = default_memory_pool();
diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64
@@ -16,7 +16,7 @@ FROM quay.io/pypa/manylinux1_x86_64:latest
 RUN yum install -y flex openssl-devel
 
 WORKDIR /
-RUN wget http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
+RUN wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz
 RUN tar xf boost_1_60_0.tar.gz
 WORKDIR /boost_1_60_0
 RUN ./bootstrap.sh

Original file line number	Diff line number	Diff line change
`@@ -67,11 +67,14 @@ TEST_F(TestBuffer, Resize) {`
`67`	`67`	`}`
`68`	`68`
`69`	`69`	`TEST_F(TestBuffer, ResizeOOM) {`
	`70`	`+ // This test doesn't play nice with AddressSanitizer`
	`71`	`+#ifndef ADDRESS_SANITIZER`
`70`	`72`	`// realloc fails, even though there may be no explicit limit`
`71`	`73`	`PoolBuffer buf;`
`72`	`74`	`ASSERT_OK(buf.Resize(100));`
`73`	`75`	`int64_t to_alloc = std::numeric_limits<int64_t>::max();`
`74`	`76`	`ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));`
	`77`	`+#endif`
`75`	`78`	`}`
`76`	`79`
`77`	`80`	`TEST_F(TestBuffer, EqualsWithSameContent) {`
Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,9 @@ TEST_F(TestDefaultMemoryPool, MemoryTracking) {`
`32`	`32`	`}`
`33`	`33`
`34`	`34`	`TEST_F(TestDefaultMemoryPool, OOM) {`
	`35`	`+#ifndef ADDRESS_SANITIZER`
`35`	`36`	`this->TestOOM();`
	`37`	`+#endif`
`36`	`38`	`}`
`37`	`39`
`38`	`40`	`TEST_F(TestDefaultMemoryPool, Reallocate) {`
`@@ -41,7 +43,7 @@ TEST_F(TestDefaultMemoryPool, Reallocate) {`
`41`	`43`
`42`	`44`	`// Death tests and valgrind are known to not play well 100% of the time. See`
`43`	`45`	`// googletest documentation`
`44`		`-#ifndef ARROW_VALGRIND`
	`46`	`+#if !(defined(ARROW_VALGRIND) \|\| defined(ADDRESS_SANITIZER))`
`45`	`47`
`46`	`48`	`TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {`
`47`	`49`	`MemoryPool* pool = default_memory_pool();`