Skip to content

Commit 16605ef

Browse files
xta0facebook-github-bot
xta0
authored and
facebook-github-bot
committed
Nightly build for for iOS (#26074)
Summary: Pull Request resolved: #26074 ### Summary This PR creates a nightly job for iOS builds. The job will generate a couple of static libraries that contains three architectures(x86, arm64, armv7s) and upload them to AWS s3. ### Note The test phase in this job is missing right now, meaning if there is a linking error, we won't be able to know it. To add the test jobs, we have to put a dummy test App in the repo and manually link the libraries to the app after the build finishes. This will be done in the next following PRs Test Plan: Imported from OSS Differential Revision: D17363066 Pulled By: xta0 fbshipit-source-id: 5beeb4263af5722f0a852297023f37aaea9ba4b1
1 parent 8c46061 commit 16605ef

File tree

9 files changed

+239
-5
lines changed

9 files changed

+239
-5
lines changed

‎.circleci/config.yml‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,8 @@ pytorch_ios_params: &pytorch_ios_params
202202
IOS_PLATFORM: << parameters.ios_platform >>
203203
USE_NNPACK: << parameters.use_nnpack >>
204204

205+
206+
205207
caffe2_params: &caffe2_params
206208
parameters:
207209
build_environment:
@@ -803,7 +805,49 @@ jobs:
803805
cat "$script"
804806
source "$script"
805807
806-
808+
binary_ios_build:
809+
<<: *pytorch_ios_params
810+
macos:
811+
xcode: "10.2.1"
812+
steps:
813+
- attach_workspace:
814+
at: ~/workspace
815+
- run:
816+
<<: *should_run_job
817+
- checkout
818+
- run:
819+
<<: *ios_brew_update
820+
- run:
821+
name: Build
822+
contxt: org-member
823+
no_output_timeout: "1h"
824+
command: |
825+
script="/Users/distiller/project/.circleci/scripts/binary_ios_build.sh"
826+
cat "$script"
827+
source "$script"
828+
- persist_to_workspace:
829+
root: /Users/distiller/workspace/
830+
paths: ios
831+
832+
binary_ios_upload:
833+
<<: *pytorch_ios_params
834+
macos:
835+
xcode: "10.2.1"
836+
steps:
837+
- attach_workspace:
838+
at: ~/workspace
839+
- run:
840+
<<: *should_run_job
841+
- checkout
842+
- run:
843+
<<: *ios_brew_update
844+
- run:
845+
name: Upload
846+
no_output_timeout: "1h"
847+
command: |
848+
script="/Users/distiller/project/.circleci/scripts/binary_ios_upload.sh"
849+
cat "$script"
850+
source "$script"
807851
setup:
808852
docker:
809853
- image: circleci/python:3.7.3
@@ -1818,7 +1862,7 @@ workflows:
18181862
- pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build
18191863
- pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build
18201864
- pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build
1821-
# Pytorch iOS builds
1865+
# Pytorch iOS PR builds
18221866
- pytorch_ios_build:
18231867
name: pytorch_ios_10_2_1_x86_64_build
18241868
build_environment: "pytorch-ios-10.2.1-x86_64_build"
@@ -1832,7 +1876,6 @@ workflows:
18321876
ios_arch: "arm64"
18331877
requires:
18341878
- setup
1835-
18361879
- caffe2_linux_build:
18371880
name: caffe2_py2_gcc4_8_ubuntu14_04_build
18381881
requires:
@@ -2978,6 +3021,36 @@ workflows:
29783021
build_environment: "libtorch 2.7 cpu"
29793022
requires:
29803023
- setup
3024+
# Pytorch iOS binary builds
3025+
- binary_ios_build:
3026+
name: pytorch_ios_10_2_1_nightly_x86_64_build
3027+
build_environment: "libtorch-ios-10.2.1-nightly-x86_64-build"
3028+
ios_platform: "SIMULATOR"
3029+
ios_arch: "x86_64"
3030+
requires:
3031+
- setup
3032+
- binary_ios_build:
3033+
name: pytorch_ios_10_2_1_nigthly_arm64_build
3034+
build_environment: "libtorch-ios-10.2.1-nightly-arm64-build"
3035+
ios_arch: "arm64"
3036+
ios_platform: "OS"
3037+
requires:
3038+
- setup
3039+
- binary_ios_build:
3040+
name: pytorch_ios_10_2_1_nightly_armv7s_build
3041+
build_environment: "libtorch-ios-10.2.1-nightly-armv7s-build"
3042+
ios_arch: "armv7s"
3043+
ios_platform: "OS"
3044+
requires:
3045+
- setup
3046+
- binary_ios_upload:
3047+
build_environment: "libtorch-ios-10.2.1-nightly-binary-build-upload"
3048+
context: org-member
3049+
requires:
3050+
- setup
3051+
- pytorch_ios_10_2_1_nightly_x86_64_build
3052+
- pytorch_ios_10_2_1_nigthly_arm64_build
3053+
- pytorch_ios_10_2_1_nightly_armv7s_build
29813054
##############################################################################
29823055
# Nightly tests
29833056
##############################################################################

‎.circleci/generate_config_yml.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ def write(self, output_filehandle):
9999
Treegen(binary_build_definitions.add_binary_smoke_test_jobs, 1),
100100
Header("Daily binary build trigger"),
101101
Treegen(binary_build_definitions.add_binary_build_jobs, 1),
102+
File("workflows-nightly-ios-binary-builds.yml"),
102103
Header("Nightly tests"),
103104
Listgen(binary_build_definitions.get_nightly_tests, 3),
104105
File("workflows-nightly-uploads-header.yml"),

‎.circleci/scripts/binary_ios_build.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
echo ""
5+
echo "PWD: ${PWD}"
6+
WORKSPACE=/Users/distiller/workspace
7+
PROJ_ROOT=/Users/distiller/project
8+
export TCLLIBPATH="/usr/local/lib"
9+
# Install conda
10+
curl -o ~/Downloads/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
11+
chmod +x ~/Downloads/conda.sh
12+
/bin/bash ~/Downloads/conda.sh -b -p ~/anaconda
13+
export PATH="~/anaconda/bin:${PATH}"
14+
source ~/anaconda/bin/activate
15+
# Install dependencies
16+
conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing requests
17+
export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
18+
# sync submodules
19+
cd ${PROJ_ROOT}
20+
git submodule sync
21+
git submodule update --init --recursive
22+
# run build script
23+
chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh
24+
echo "########################################################"
25+
cat ${PROJ_ROOT}/scripts/build_ios.sh
26+
echo "########################################################"
27+
echo "IOS_ARCH: ${IOS_ARCH}"
28+
echo "IOS_PLATFORM: ${IOS_PLATFORM}"
29+
export BUILD_PYTORCH_MOBILE=1
30+
export IOS_ARCH=${IOS_ARCH}
31+
export IOS_PLATFORM=${IOS_PLATFORM}
32+
unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts
33+
#store the binary
34+
cd ${WORKSPACE}
35+
DEST_DIR=${WORKSPACE}/ios
36+
mkdir -p ${DEST_DIR}
37+
cp -R ${PROJ_ROOT}/build_ios/install ${DEST_DIR}
38+
mv ${DEST_DIR}/install ${DEST_DIR}/${IOS_ARCH}

‎.circleci/scripts/binary_ios_upload.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
echo ""
5+
echo "PWD: $(pwd)"
6+
WORKSPACE=/Users/distiller/workspace
7+
PROJ_ROOT=/Users/distiller/project
8+
ARTIFACTS_DIR=${WORKSPACE}/ios
9+
ls ${ARTIFACTS_DIR}
10+
ZIP_DIR=${WORKSPACE}/zip
11+
mkdir -p ${ZIP_DIR}/install/lib
12+
mkdir -p ${ZIP_DIR}/src
13+
# copy header files
14+
cp -R ${ARTIFACTS_DIR}/arm64/include ${ZIP_DIR}/install/
15+
# build a FAT bianry
16+
cd ${ZIP_DIR}/install/lib
17+
target_libs=(libc10.a libclog.a libcpuinfo.a libqnnpack.a libtorch.a)
18+
for lib in ${target_libs[*]}
19+
do
20+
libs=(${ARTIFACTS_DIR}/x86_64/lib/${lib} ${ARTIFACTS_DIR}/arm64/lib/${lib} ${ARTIFACTS_DIR}/armv7s/lib/${lib} )
21+
lipo -create "${libs[@]}" -o ${ZIP_DIR}/install/lib/${lib}
22+
done
23+
# for nnpack, we only support arm64/armv7s build
24+
lipo -create ${ARTIFACTS_DIR}/arm64/lib/libnnpack.a ${ARTIFACTS_DIR}/armv7s/lib/libnnpack.a -o ${ZIP_DIR}/install/lib/libnnpack.a
25+
lipo -i ${ZIP_DIR}/install/lib/*.a
26+
# copy the umbrella header and license
27+
cp ${PROJ_ROOT}/ios/LibTorch.h ${ZIP_DIR}/src/
28+
cp ${PROJ_ROOT}/LICENSE ${ZIP_DIR}/
29+
# zip the library
30+
ZIPFILE=libtorch_ios_nightly_build.zip
31+
cd ${ZIP_DIR}
32+
#for testing
33+
touch version.txt
34+
echo $(date +%s) > version.txt
35+
zip -r ${ZIPFILE} install src version.txt LICENSE
36+
# upload to aws
37+
brew install awscli
38+
set +x
39+
export AWS_ACCESS_KEY_ID=${AWS_S3_ACCESS_KEY_FOR_PYTORCH_BINARY_UPLOAD}
40+
export AWS_SECRET_ACCESS_KEY=${AWS_S3_ACCESS_SECRET_FOR_PYTORCH_BINARY_UPLOAD}
41+
set +x
42+
# echo "AWS KEY: ${AWS_ACCESS_KEY_ID}"
43+
# echo "AWS SECRET: ${AWS_SECRET_ACCESS_KEY}"
44+
aws s3 cp ${ZIPFILE} s3://ossci-ios-build/ --acl public-read

‎.circleci/scripts/should_run_job.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@
3535
'manywheel 2.7mu cpu devtoolset7',
3636
'libtorch 2.7m cpu devtoolset7',
3737
'libtorch 2.7m cpu gcc5.4_cxx11-abi',
38+
'libtorch-ios-10.2.1-nightly-x86_64-build',
39+
'libtorch-ios-10.2.1-nightly-arm64-build',
40+
'libtorch-ios-10.2.1-nightly-armv7s-build',
41+
'libtorch-ios-10.2.1-nightly-binary-build-upload',
3842

3943
# Caffe2 Android
4044
'caffe2-py2-android-ubuntu16.04',

‎.circleci/verbatim-sources/binary-job-specs.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,3 +216,46 @@
216216
cat "$script"
217217
source "$script"
218218
219+
binary_ios_build:
220+
<<: *pytorch_ios_params
221+
macos:
222+
xcode: "10.2.1"
223+
steps:
224+
- attach_workspace:
225+
at: ~/workspace
226+
- run:
227+
<<: *should_run_job
228+
- checkout
229+
- run:
230+
<<: *ios_brew_update
231+
- run:
232+
name: Build
233+
contxt: org-member
234+
no_output_timeout: "1h"
235+
command: |
236+
script="/Users/distiller/project/.circleci/scripts/binary_ios_build.sh"
237+
cat "$script"
238+
source "$script"
239+
- persist_to_workspace:
240+
root: /Users/distiller/workspace/
241+
paths: ios
242+
243+
binary_ios_upload:
244+
<<: *pytorch_ios_params
245+
macos:
246+
xcode: "10.2.1"
247+
steps:
248+
- attach_workspace:
249+
at: ~/workspace
250+
- run:
251+
<<: *should_run_job
252+
- checkout
253+
- run:
254+
<<: *ios_brew_update
255+
- run:
256+
name: Upload
257+
no_output_timeout: "1h"
258+
command: |
259+
script="/Users/distiller/project/.circleci/scripts/binary_ios_upload.sh"
260+
cat "$script"
261+
source "$script"

‎.circleci/verbatim-sources/pytorch-build-params.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,5 @@ pytorch_ios_params: &pytorch_ios_params
3838
IOS_PLATFORM: << parameters.ios_platform >>
3939
USE_NNPACK: << parameters.use_nnpack >>
4040

41+
42+

‎.circleci/verbatim-sources/workflows-nightly-ios-binary-builds.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Pytorch iOS binary builds
2+
- binary_ios_build:
3+
name: pytorch_ios_10_2_1_nightly_x86_64_build
4+
build_environment: "libtorch-ios-10.2.1-nightly-x86_64-build"
5+
ios_platform: "SIMULATOR"
6+
ios_arch: "x86_64"
7+
requires:
8+
- setup
9+
- binary_ios_build:
10+
name: pytorch_ios_10_2_1_nigthly_arm64_build
11+
build_environment: "libtorch-ios-10.2.1-nightly-arm64-build"
12+
ios_arch: "arm64"
13+
ios_platform: "OS"
14+
requires:
15+
- setup
16+
- binary_ios_build:
17+
name: pytorch_ios_10_2_1_nightly_armv7s_build
18+
build_environment: "libtorch-ios-10.2.1-nightly-armv7s-build"
19+
ios_arch: "armv7s"
20+
ios_platform: "OS"
21+
requires:
22+
- setup
23+
- binary_ios_upload:
24+
build_environment: "libtorch-ios-10.2.1-nightly-binary-build-upload"
25+
context: org-member
26+
requires:
27+
- setup
28+
- pytorch_ios_10_2_1_nightly_x86_64_build
29+
- pytorch_ios_10_2_1_nigthly_arm64_build
30+
- pytorch_ios_10_2_1_nightly_armv7s_build

‎.circleci/verbatim-sources/workflows-pytorch-ios-builds.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Pytorch iOS builds
1+
# Pytorch iOS PR builds
22
- pytorch_ios_build:
33
name: pytorch_ios_10_2_1_x86_64_build
44
build_environment: "pytorch-ios-10.2.1-x86_64_build"
@@ -12,4 +12,3 @@
1212
ios_arch: "arm64"
1313
requires:
1414
- setup
15-

0 commit comments

Comments
 (0)