Add Linux Jenkins scripts to PyTorch repo. (#4910)

ezyang · web-flow · commit e93ece90a5c9 · 2018-01-29T15:08:06.000-05:00
Putting these scripts here has a few benefits:

1. PyTorch developers can easily update the scripts without
having to ask for permissions to ossci-job-dsl

2. You can test changes in the scripts by opening a PR to
PyTorch (functionality is ossci-job-dsl is not easily testable.)

3. If you get one of our stock Docker images, you can run these scripts
to trigger a build identical to what would occur in Jenkins (not
entirely true yet, but we can make it so.)

Signed-off-by: Edward Z. Yang &lt;ezyang@fb.com&gt;
diff --git a/.jenkins/build.sh b/.jenkins/build.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Required environment variables:
+#   $JOB_NAME
+#   $PYTHON_VERSION
+#   $GCC_VERSION
+#
+# TODO: change this script to make use of $BUILD_ENVIRONMENT,
+# which we can hard code into Docker images and then these scripts
+# will work out of the box without having to set any env vars.
+
+set -ex
+
+export PATH=/opt/conda/bin:$PATH
+
+if [[ "$JOB_NAME" != *cuda* ]]; then
+   export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH
+   export LD_LIBRARY_PATH=/opt/python/${PYTHON_VERSION}/lib:$LD_LIBRARY_PATH
+   export CC="ccache /usr/bin/gcc-${GCC_VERSION}"
+   export CXX="ccache /usr/bin/g++-${GCC_VERSION}"
+else
+   # CMake should use ccache symlink for nvcc
+   export CUDA_NVCC_EXECUTABLE=/usr/local/bin/nvcc
+   # The ccache wrapper should be able to find the real nvcc
+   export PATH=/usr/local/cuda/bin:$PATH
+
+   # Add CUDA stub/real path to loader path
+   export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
+   export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+
+   # Build for Maxwell
+   export TORCH_CUDA_ARCH_LIST="Maxwell"
+   export TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
+fi
+
+echo "Python Version:"
+which python
+python --version
+
+pip install -r requirements.txt || true
+
+# This token is used by a parser on Jenkins logs for determining
+# if a failure is a legitimate problem, or a problem with the build
+# system; to find out more, grep for this string in ossci-job-dsl.
+echo "ENTERED_USER_LAND"
+
+time python setup.py install
+
+if [[ "$JOB_NAME" != *cuda* ]]; then
+   echo "Testing ATen"
+   time tools/run_aten_tests.sh
+fi
+
+echo "EXITED_USER_LAND"
diff --git a/.jenkins/test.sh b/.jenkins/test.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Required environment variables:
+#   $JOB_NAME
+#   $PYTHON_VERSION
+#   $GCC_VERSION
+
+export PATH=/opt/conda/bin:$PATH
+
+if [[ "$JOB_NAME" == *cuda* ]]; then
+   export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
+   export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+else
+   export PATH=/opt/python/${PYTHON_VERSION}/bin:$PATH
+   export LD_LIBRARY_PATH=/opt/python/${PYTHON_VERSION}/lib:$LD_LIBRARY_PATH
+
+   # NB: setup.py chokes on a setting of CC='ccache gcc' (two words),
+   # so we created a symlinked binary that we can pass as CC in one word
+   mkdir ./ccache
+   ln -sf "$(which ccache)" ./ccache/gcc-${GCC_VERSION}
+   ln -sf "$(which ccache)" ./ccache/g++-${GCC_VERSION}
+   export CC="$PWD/ccache/gcc-${GCC_VERSION}"
+   export CXX="$PWD/ccache/g++-${GCC_VERSION}"
+fi
+
+echo "Installing torchvision at branch master"
+rm -rf vision
+git clone https://github.com/pytorch/vision --quiet
+if [[ "$JOB_NAME" == *cuda* ]]; then
+   conda install -y pillow
+else
+   pip install pillow
+fi
+
+echo "ENTERED_USER_LAND"
+
+echo "Testing pytorch"
+export OMP_NUM_THREADS=4
+export MKL_NUM_THREADS=4
+time test/run_test.sh
+
+pushd vision
+time python setup.py install
+popd
+
+echo "EXITED_USER_LAND"
