Switch to out-source builds for LibTorch

peterjc123 · facebook-github-bot · commit 794ee6d00ccf · 2019-06-14T21:00:18.000-07:00
Summary: Pull Request resolved: #21772 Differential Revision: D15839332 Pulled By: yf225 fbshipit-source-id: 017cf61c5682c6a8ffeaf2ca952e1418c27be30e
diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh
@@ -163,13 +163,13 @@ test_libtorch() {
     python test/cpp/jit/tests_setup.py setup
     CPP_BUILD="$PWD/../cpp-build"
     if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
-      "$CPP_BUILD"/caffe2/bin/test_jit
+      "$CPP_BUILD"/caffe2/build/bin/test_jit
     else
-      "$CPP_BUILD"/caffe2/bin/test_jit "[cpu]"
+      "$CPP_BUILD"/caffe2/build/bin/test_jit "[cpu]"
     fi
     python test/cpp/jit/tests_setup.py shutdown
     python tools/download_mnist.py --quiet -d test/cpp/api/mnist
-    OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" "$CPP_BUILD"/caffe2/bin/test_api
+    OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" "$CPP_BUILD"/caffe2/build/bin/test_api
     assert_git_not_dirty
   fi
 }
diff --git a/.jenkins/pytorch/win-test-helpers/build_pytorch.bat b/.jenkins/pytorch/win-test-helpers/build_pytorch.bat
@@ -102,7 +102,6 @@ if not "%USE_CUDA%"=="0" (
     if "%BUILD_ENVIRONMENT%"=="" (
       echo NOTE: To run `import torch`, please make sure to activate the conda environment by running `call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3` in Command Prompt before running Git Bash.
     ) else (
-      mv %CD%\build\bin\test_api.exe %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch\lib
       7z a %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\caffe2 && python %SCRIPT_HELPERS_DIR%\upload_image.py %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z
     )
   )
diff --git a/.jenkins/pytorch/win-test-helpers/test_libtorch.bat b/.jenkins/pytorch/win-test-helpers/test_libtorch.bat
@@ -1,9 +1,7 @@
 call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
 
-dir
-dir %TMP_DIR_WIN%\build
-dir %TMP_DIR_WIN%\build\torch
-dir %TMP_DIR_WIN%\build\torch\lib
-cd %TMP_DIR_WIN%\build\torch\lib
+cd %TMP_DIR_WIN%\build\torch\bin
 set PATH=C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64;%TMP_DIR_WIN%\build\torch\lib;%PATH%
 test_api.exe --gtest_filter="-IntegrationTest.MNIST*"
+
+if errorlevel 1 exit /b 1
diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
@@ -1040,10 +1040,10 @@ if (BUILD_TEST)
     add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
     if (INSTALL_TEST)
       install(TARGETS ${test_name} DESTINATION test)
-        # Install PDB files for MSVC builds
-        if (MSVC AND BUILD_SHARED_LIBS)
-          install(FILES $<TARGET_PDB_FILE:${test_name}> DESTINATION test OPTIONAL)
-        endif()
+      # Install PDB files for MSVC builds
+      if (MSVC AND BUILD_SHARED_LIBS)
+        install(FILES $<TARGET_PDB_FILE:${test_name}> DESTINATION test OPTIONAL)
+      endif()
     endif()
   endforeach()
 
diff --git a/cmake/public/cuda.cmake b/cmake/public/cuda.cmake
@@ -47,6 +47,14 @@ if(CUDA_FOUND)
   # compiling with, e.g., if a ccache nvcc is fed to us by CUDA_NVCC_EXECUTABLE
   # but the PATH is not consistent with CUDA_HOME.  It's better safe
   # than sorry: make sure everything is consistent.
+  if(MSVC AND CMAKE_GENERATOR MATCHES "Visual Studio")
+    # When using Visual Studio, it attempts to lock the whole binary dir when
+    # `try_run` is called, which will cause the build to fail.
+    string(RANDOM BUILD_SUFFIX)
+    set(PROJECT_RANDOM_BINARY_DIR "${PROJECT_BINARY_DIR}/${BUILD_SUFFIX}")
+  else()
+    set(PROJECT_RANDOM_BINARY_DIR "${PROJECT_BINARY_DIR}")
+  endif()
   set(file "${PROJECT_BINARY_DIR}/detect_cuda_version.cc")
   file(WRITE ${file} ""
     "#include <cuda.h>\n"
@@ -56,7 +64,7 @@ if(CUDA_FOUND)
     "  return 0;\n"
     "}\n"
     )
-  try_run(run_result compile_result ${PROJECT_BINARY_DIR} ${file}
+  try_run(run_result compile_result ${PROJECT_RANDOM_BINARY_DIR} ${file}
     CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CUDA_INCLUDE_DIRS}"
     LINK_LIBRARIES ${CUDA_LIBRARIES}
     RUN_OUTPUT_VARIABLE cuda_version_from_header
diff --git a/test/cpp/api/CMakeLists.txt b/test/cpp/api/CMakeLists.txt
@@ -57,3 +57,11 @@ if (NOT MSVC)
       -Wno-unused-but-set-parameter)
   endif()
 endif()
+
+if (INSTALL_TEST)
+  install(TARGETS test_api DESTINATION bin)
+  # Install PDB files for MSVC builds
+  if (MSVC AND BUILD_SHARED_LIBS)
+    install(FILES $<TARGET_PDB_FILE:test_api> DESTINATION bin OPTIONAL)
+  endif()
+endif()
diff --git a/test/cpp/jit/CMakeLists.txt b/test/cpp/jit/CMakeLists.txt
@@ -17,3 +17,11 @@ if (USE_CUDA)
 
   target_compile_definitions(test_jit PRIVATE USE_CUDA)
 endif()
+
+if (INSTALL_TEST)
+  install(TARGETS test_jit DESTINATION bin)
+  # Install PDB files for MSVC builds
+  if (MSVC AND BUILD_SHARED_LIBS)
+    install(FILES $<TARGET_PDB_FILE:test_jit> DESTINATION bin OPTIONAL)
+  endif()
+endif()
diff --git a/tools/build_libtorch.py b/tools/build_libtorch.py
@@ -20,4 +20,4 @@
     options = parser.parse_args()
 
     build_caffe2(version=None, cmake_python_library=None, build_python=False,
-                 rerun_cmake=True, cmake_only=False, cmake=CMake('.'))
+                 rerun_cmake=True, cmake_only=False, cmake=CMake('build'))

Original file line number	Diff line number	Diff line change
`@@ -102,7 +102,6 @@ if not "%USE_CUDA%"=="0" (`
`102`	`102`	`if "%BUILD_ENVIRONMENT%"=="" (`
`103`	`103`	echo NOTE: To run `import torch`, please make sure to activate the conda environment by running `call %CONDA_PARENT_DIR%\Miniconda3\Scripts\activate.bat %CONDA_PARENT_DIR%\Miniconda3` in Command Prompt before running Git Bash.
`104`	`104`	`) else (`
`105`		`- mv %CD%\build\bin\test_api.exe %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch\lib`
`106`	`105`	`7z a %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\torch %CONDA_PARENT_DIR%\Miniconda3\Lib\site-packages\caffe2 && python %SCRIPT_HELPERS_DIR%\upload_image.py %TMP_DIR_WIN%\%IMAGE_COMMIT_TAG%.7z`
`107`	`106`	`)`
`108`	`107`	`)`