ARROW-7374: [Dev] [C++] Fix cuda-cpp docker build

pitrou · kou · commit bce0899fd5ce · 2019-12-17T06:40:32.000+09:00
Also add a CUDA 9.1 Dockerfile for Ubuntu 18.04 hosts. (official Ubuntu 18.04 packages don't provide CUDA 10.0, only 9.1) Closes apache#6016 from pitrou/ARROW-7374-docker-cuda-cpp and squashes the following commits: 6fa82b2 <Antoine Pitrou> Apply @kou's proposed fix 3e65972 <Antoine Pitrou> ARROW-7374: Fix cuda-cpp docker build Authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
diff --git a/.env b/.env
@@ -20,7 +20,7 @@
 
 REPO=apache/arrow-dev
 ARCH=amd64
-CUDA=10.0
+CUDA=9.1
 DEBIAN=10
 UBUNTU=18.04
 FEDORA=29
diff --git a/ci/docker/cuda-10.0-cpp.dockerfile b/ci/docker/cuda-10.0-cpp.dockerfile
@@ -59,6 +59,7 @@ RUN apt-get update -y -q && \
       ninja-build \
       pkg-config \
       protobuf-compiler \
+      python-minimal \
       rapidjson-dev \
       thrift-compiler \
       tzdata && \
@@ -72,17 +73,23 @@ RUN apt-get update -y -q && \
 # - libgtest-dev only provide sources
 # - libprotobuf-dev only provide sources
 # - thrift is too old
-ENV ARROW_CUDA=ON \
+ENV ARROW_BUILD_STATIC=OFF \
+    ARROW_BUILD_TESTS=ON \
+    ARROW_COMPUTE=OFF \
+    ARROW_CSV=OFF \
+    ARROW_CUDA=ON \
+    ARROW_DATASET=OFF \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
+    ARROW_FILESYSTEM=OFF \
     ARROW_FLIGHT=OFF \
     ARROW_HOME=/usr/local \
     ARROW_INSTALL_NAME_RPATH=OFF \
     ARROW_NO_DEPRECATED_API=ON \
     ARROW_PLASMA=ON \
-    ARROW_USE_ASAN=ON \
     ARROW_USE_CCACHE=ON \
-    ARROW_USE_UBSAN=ON \
+    CUDA_LIB_PATH=${LIBRARY_PATH} \
     GTest_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PATH=/usr/lib/ccache/:$PATH \
-    Thrift_SOURCE=BUNDLED
+    Thrift_SOURCE=BUNDLED \
+    LD_LIBRARY_PATH=/usr/local/cuda/lib64/
diff --git a/ci/docker/cuda-9.1-cpp.dockerfile b/ci/docker/cuda-9.1-cpp.dockerfile
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM nvidia/cuda:9.1-devel-ubuntu16.04
+
+# pipefail is enabled for proper error detection in the `wget | apt-key add`
+# step
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+      wget software-properties-common && \
+      apt-get clean && rm -rf /var/lib/apt/lists*
+
+# Installs C++ toolchain and dependencies
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+      autoconf \
+      bison \
+      ca-certificates \
+      ccache \
+      cmake \
+      flex \
+      g++ \
+      gcc \
+      git \
+      libboost-filesystem-dev \
+      libboost-regex-dev \
+      libboost-system-dev \
+      libbrotli-dev \
+      libbz2-dev \
+      libgflags-dev \
+      libgoogle-glog-dev \
+      liblz4-dev \
+      liblzma-dev \
+      libprotobuf-dev \
+      libprotoc-dev \
+      libre2-dev \
+      libsnappy-dev \
+      libssl-dev \
+      libzstd-dev \
+      ninja-build \
+      pkg-config \
+      protobuf-compiler \
+      python-minimal \
+      rapidjson-dev \
+      thrift-compiler \
+      tzdata && \
+      apt-get clean && rm -rf /var/lib/apt/lists*
+
+# Prioritize system packages and local installation
+# The following dependencies will be downloaded due to missing/invalid packages
+# provided by the distribution:
+# - libc-ares-dev does not install CMake config files
+# - flatbuffer is not packaged
+# - libgtest-dev only provide sources
+# - libprotobuf-dev only provide sources
+# - thrift is too old
+ENV ARROW_BUILD_STATIC=OFF \
+    ARROW_BUILD_TESTS=ON \
+    ARROW_COMPUTE=OFF \
+    ARROW_CSV=OFF \
+    ARROW_CUDA=ON \
+    ARROW_DATASET=OFF \
+    ARROW_DEPENDENCY_SOURCE=SYSTEM \
+    ARROW_FILESYSTEM=OFF \
+    ARROW_FLIGHT=OFF \
+    ARROW_HOME=/usr/local \
+    ARROW_INSTALL_NAME_RPATH=OFF \
+    ARROW_NO_DEPRECATED_API=ON \
+    ARROW_PLASMA=ON \
+    ARROW_USE_CCACHE=ON \
+    GTest_SOURCE=BUNDLED \
+    ORC_SOURCE=BUNDLED \
+    PATH=/usr/lib/ccache/:$PATH \
+    Thrift_SOURCE=BUNDLED
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -31,13 +31,14 @@ volumes:
   # like ARM64v8 with debian 10, then the appropiate parametrized volume name
   # must be added the the list below: arm64v8-debian-10-cache
   amd64-conda-cache:
+  amd64-cuda-9.1-cache:
   amd64-cuda-10.0-cache:
   amd64-debian-9-cache:
   amd64-debian-10-cache:
-  amd64-ubuntu-18.04-cache:
-  amd64-ubuntu-16.04-cache:
-  amd64-ubuntu-14.04-cache:
   amd64-fedora-29-cache:
+  amd64-ubuntu-14.04-cache:
+  amd64-ubuntu-16.04-cache:
+  amd64-ubuntu-18.04-cache:
   maven-cache:
 
 x-ccache: &ccache
@@ -93,9 +94,11 @@ services:
     # Usage:
     #   docker-compose build cuda-cpp
     #   docker-compose run --rm cuda-cpp
+    # Also need to edit the host docker configuration as follows:
+    #   https://github.com/docker/compose/issues/6691#issuecomment-561504928
     # Parameters:
     #   ARCH: amd64
-    #   CUDA: 8.0, 10.0, ...
+    #   CUDA: 9.1, 10.0
     image: ${REPO}:${ARCH}-cuda-${CUDA}-cpp
     build:
       context: .
