ARROW-11377: [C++][CI] Add Thread Sanitizer nightly build

pitrou · pitrou · commit 2cb8ef0825f2 · 2021-01-26T14:33:38.000+01:00
Closes apache#9317 from pitrou/ARROW-11377-tsan-nightly Authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Antoine Pitrou <antoine@python.org>
diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile
@@ -112,6 +112,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_PLASMA=ON \
     ARROW_USE_ASAN=OFF \
     ARROW_USE_CCACHE=ON \
+    ARROW_USE_TSAN=OFF \
     ARROW_USE_UBSAN=OFF \
     ARROW_WITH_BROTLI=ON \
     ARROW_WITH_BZ2=ON \
diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh
@@ -92,6 +92,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
       -DARROW_USE_LD_GOLD=${ARROW_USE_LD_GOLD:-OFF} \
       -DARROW_USE_PRECOMPILED_HEADERS=${ARROW_USE_PRECOMPILED_HEADERS:-OFF} \
       -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \
+      -DARROW_USE_TSAN=${ARROW_USE_TSAN:-OFF} \
       -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \
       -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \
       -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \
diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh
@@ -65,14 +65,10 @@ function setup_sanitizers() {
 
   # Configure TSAN (ignored if this isn't a TSAN build).
   #
-  # Deadlock detection (new in clang 3.5) is disabled because:
-  # 1. The clang 3.5 deadlock detector crashes in some unit tests. It
-  #    needs compiler-rt commits c4c3dfd, 9a8efe3, and possibly others.
-  # 2. Many unit tests report lock-order-inversion warnings; they should be
-  #    fixed before reenabling the detector.
-  TSAN_OPTIONS="$TSAN_OPTIONS detect_deadlocks=0"
   TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$ROOT/build-support/tsan-suppressions.txt"
   TSAN_OPTIONS="$TSAN_OPTIONS history_size=7"
+  # Some tests deliberately fail allocating memory
+  TSAN_OPTIONS="$TSAN_OPTIONS allocator_may_return_null=1"
   export TSAN_OPTIONS
 
   UBSAN_OPTIONS="$UBSAN_OPTIONS print_stacktrace=1"
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -1580,32 +1580,51 @@ tasks:
     ci: circle
     template: docker-tests/circle.linux.yml
     params:
+      env:
+        UBUNTU: 18.04
       run: "-e ARROW_BUILD_TYPE=release ubuntu-cpp"
 
   test-ubuntu-18.04-cpp-static:
     ci: circle
     template: docker-tests/circle.linux.yml
     params:
+      env:
+        UBUNTU: 18.04
       run: "-e ARROW_BUILD_SHARED=OFF -e ARROW_BUILD_STATIC=ON -e ARROW_TEST_LINKAGE=static ubuntu-cpp"
 
   test-ubuntu-20.04-cpp:
     ci: github
     template: docker-tests/github.linux.yml
     params:
+      env:
+        UBUNTU: 20.04
       run: ubuntu-cpp
 
   test-ubuntu-20.04-cpp-14:
     ci: github
     template: docker-tests/github.linux.yml
     params:
+      env:
+        UBUNTU: 20.04
       run: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD=14 ubuntu-cpp"
 
   test-ubuntu-20.04-cpp-17:
     ci: github
     template: docker-tests/github.linux.yml
     params:
+      env:
+        UBUNTU: 20.04
       run: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD=17 ubuntu-cpp"
 
+  test-ubuntu-20.04-cpp-thread-sanitizer:
+    ci: github
+    template: docker-tests/github.linux.yml
+    params:
+      env:
+        CLANG_TOOLS: 11
+        UBUNTU: 20.04
+      run: ubuntu-cpp-thread-sanitizer
+
   test-debian-c-glib:
     ci: circle
     template: docker-tests/circle.linux.yml
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -113,6 +113,7 @@ x-hierarchy:
     - ubuntu-cuda-python
   - ubuntu-csharp
   - ubuntu-cpp-sanitizer
+  - ubuntu-cpp-thread-sanitizer
   - ubuntu-r-sanitizer
   - python-sdist
   - r
@@ -350,6 +351,37 @@ services:
       utf8proc_SOURCE: "BUNDLED"
     command: *cpp-command
 
+  ubuntu-cpp-thread-sanitizer:
+    # Usage:
+    #   docker-compose build ubuntu-cpp-thread-sanitizer
+    #   docker-compose run --rm ubuntu-cpp-thread-sanitizer
+    # Parameters:
+    #   ARCH: amd64, arm64v8, ...
+    #   UBUNTU: 16.04, 18.04, 20.04
+    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
+    build:
+      context: .
+      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
+      args:
+        arch: ${ARCH}
+        clang_tools: ${CLANG_TOOLS}
+        llvm: ${LLVM}
+    shm_size: *shm-size
+    volumes: *ubuntu-volumes
+    environment:
+      <<: *ccache
+      CC: clang-${CLANG_TOOLS}
+      CXX: clang++-${CLANG_TOOLS}
+      ARROW_ENABLE_TIMING_TESTS:  # inherit
+      ARROW_DATASET: "ON"
+      ARROW_JEMALLOC: "OFF"
+      ARROW_ORC: "OFF"
+      ARROW_S3: "OFF"
+      ARROW_USE_TSAN: "ON"
+    command: *cpp-command
+
   fedora-cpp:
     # Usage:
     #   docker-compose build fedora-cpp
