Skip to content

Commit e22095b

Browse files
ezyangezyang
authored
Add some more builder scripts from ossci-job-dsl (#4945)
* Add some more builder scripts from ossci-job-dsl Signed-off-by: Edward Z. Yang <ezyang@fb.com> * Relax precision requirement on test_Upsample_trilinear_scale_3d_cuda Partially addresses #5006. Signed-off-by: Edward Z. Yang <ezyang@fb.com>
1 parent bf3655a commit e22095b

File tree

7 files changed

+527
-6
lines changed

7 files changed

+527
-6
lines changed

‎.jenkins/macos-build-test.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
# Set up conda environment
6+
curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o $PWD/miniconda3.sh
7+
rm -rf $PWD/miniconda3
8+
bash $PWD/miniconda3.sh -b -p $PWD/miniconda3
9+
export PATH="$PWD/miniconda3/bin:$PATH"
10+
source $PWD/miniconda3/bin/activate
11+
conda install -y numpy pyyaml setuptools cmake cffi ninja
12+
13+
# Build and test PyTorch
14+
git submodule update --init --recursive
15+
export CMAKE_PREFIX_PATH=$PWD/miniconda3/
16+
17+
echo "ENTERED_USER_LAND"
18+
19+
export MACOSX_DEPLOYMENT_TARGET=10.9
20+
export CXX=clang++
21+
export CC=clang
22+
python setup.py install
23+
cd test/
24+
echo "Ninja version: $(ninja --version)"
25+
sh run_test.sh
26+
echo "EXITED_USER_LAND"
27+
echo "BUILD PASSED"

‎.jenkins/short-perf-test-cpu.sh‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
export PATH=/opt/conda/bin:$PATH
6+
7+
echo "Running CPU perf test for PyTorch..."
8+
export OMP_NUM_THREADS=4
9+
export MKL_NUM_THREADS=4
10+
11+
cd test/
12+
13+
# TODO: Move this into its own file
14+
cat >compare_with_baseline.py << EOL
15+
16+
import sys
17+
import numpy
18+
from scipy import stats
19+
20+
mean_values = {
21+
"commit": "92aeca1279265d24493dc6ced7dde9a368faf048",
22+
23+
"test_cpu_speed_mini_sequence_labeler": "2.62557",
24+
"test_cpu_speed_mnist": "18.79468",
25+
}
26+
27+
sigma_values = {
28+
"commit": "92aeca1279265d24493dc6ced7dde9a368faf048",
29+
30+
"test_cpu_speed_mini_sequence_labeler": "0.12167",
31+
"test_cpu_speed_mnist": "2.37561",
32+
}
33+
34+
mean = float(mean_values[sys.argv[1]])
35+
sigma = float(sigma_values[sys.argv[1]])
36+
37+
print("population mean: ", mean)
38+
print("population sigma: ", sigma)
39+
40+
sample_data_list = sys.argv[2:]
41+
sample_data_list = [float(v.strip()) for v in sample_data_list]
42+
43+
sample_mean = numpy.mean(sample_data_list)
44+
45+
print("sample mean: ", sample_mean)
46+
47+
z_value = (sample_mean - mean) / sigma
48+
49+
print("z-value: ", z_value)
50+
51+
if z_value >= 3:
52+
raise Exception("z-value >= 3, there is 99.7% chance of perf regression.")
53+
else:
54+
print("z-value < 3, no perf regression detected.")
55+
56+
EOL
57+
58+
run_test () {
59+
mkdir test_tmp/ && cd test_tmp/
60+
$1
61+
cd .. && rm -rf test_tmp/
62+
}
63+
64+
get_runtime_of_command () {
65+
TIMEFORMAT=%R
66+
67+
# runtime=$( { time ($1 &> /dev/null); } 2>&1 1>/dev/null)
68+
runtime=$( { time $1; } 2>&1 1>/dev/null)
69+
runtime=${runtime#+++ $1}
70+
runtime=$(python -c "print($runtime)")
71+
72+
echo $runtime
73+
}
74+
75+
# Define tests
76+
test_cpu_speed_mini_sequence_labeler () {
77+
echo "Testing: mini sequence labeler, CPU"
78+
79+
curl https://gist.githubusercontent.com/yf225/40c0adb8bfb2a7b774fa266fb4bc409e/raw/20c67ebadbd75f41c6c9fd2e00b4b2562b60700a/mini_sequence_labeler.py -O
80+
curl https://gist.githubusercontent.com/yf225/592b39ca6a3fc835a4d1532fb1474d26/raw/76f57c198cb7afdc5122e413c2a3023ed024b643/wsj.pkl -O
81+
82+
SAMPLE_ARRAY=()
83+
NUM_RUNS=20
84+
85+
for (( i=1; i<=$NUM_RUNS; i++ )) do
86+
runtime=$(get_runtime_of_command "python mini_sequence_labeler.py")
87+
echo $runtime
88+
SAMPLE_ARRAY+=(${runtime})
89+
done
90+
91+
python ../compare_with_baseline.py ${FUNCNAME[0]} "${SAMPLE_ARRAY[@]}"
92+
}
93+
94+
test_cpu_speed_mnist () {
95+
echo "Testing: MNIST, CPU"
96+
97+
git clone https://github.com/yf225/examples.git -b benchmark_test
98+
99+
cd examples/mnist
100+
101+
pip install -r requirements.txt
102+
103+
# Download data
104+
python main.py --epochs 0
105+
106+
SAMPLE_ARRAY=()
107+
NUM_RUNS=20
108+
109+
for (( i=1; i<=$NUM_RUNS; i++ )) do
110+
runtime=$(get_runtime_of_command "python main.py --epochs 1 --no-log")
111+
echo $runtime
112+
SAMPLE_ARRAY+=(${runtime})
113+
done
114+
115+
cd ../..
116+
117+
python ../compare_with_baseline.py ${FUNCNAME[0]} "${SAMPLE_ARRAY[@]}"
118+
}
119+
120+
echo "ENTERED_USER_LAND"
121+
122+
# Run tests
123+
run_test test_cpu_speed_mini_sequence_labeler
124+
run_test test_cpu_speed_mnist
125+
126+
echo "EXITED_USER_LAND"

‎.jenkins/short-perf-test-gpu.sh‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
export PATH=/opt/conda/bin:$PATH
6+
7+
export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH
8+
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
9+
10+
echo "Running GPU perf test for PyTorch..."
11+
export OMP_NUM_THREADS=4
12+
export MKL_NUM_THREADS=4
13+
14+
cd test/
15+
16+
# TODO: Move this into a separate file
17+
cat >compare_with_baseline.py << EOL
18+
19+
import sys
20+
import numpy
21+
from scipy import stats
22+
23+
mean_values = {
24+
"commit": "92aeca1279265d24493dc6ced7dde9a368faf048",
25+
26+
"test_gpu_speed_mnist": "13.76155",
27+
"test_gpu_speed_word_language_model": "114.20865",
28+
"test_gpu_speed_cudnn_lstm": "4.9698",
29+
"test_gpu_speed_lstm": "5.15325",
30+
"test_gpu_speed_mlstm": "4.04270",
31+
}
32+
33+
sigma_values = {
34+
"commit": "92aeca1279265d24493dc6ced7dde9a368faf048",
35+
36+
"test_gpu_speed_mnist": "0.42087",
37+
"test_gpu_speed_word_language_model": "0.27014",
38+
"test_gpu_speed_cudnn_lstm": "0.03257",
39+
"test_gpu_speed_lstm": "0.0725",
40+
"test_gpu_speed_mlstm": "0.03276",
41+
}
42+
43+
mean = float(mean_values[sys.argv[1]])
44+
sigma = float(sigma_values[sys.argv[1]])
45+
46+
print("population mean: ", mean)
47+
print("population sigma: ", sigma)
48+
49+
sample_data_list = sys.argv[2:]
50+
sample_data_list = [float(v.strip()) for v in sample_data_list]
51+
52+
sample_mean = numpy.mean(sample_data_list)
53+
54+
print("sample mean: ", sample_mean)
55+
56+
z_value = (sample_mean - mean) / sigma
57+
58+
print("z-value: ", z_value)
59+
60+
if z_value >= 3:
61+
raise Exception("z-value >= 3, there is 99.7% chance of perf regression.")
62+
else:
63+
print("z-value < 3, no perf regression detected.")
64+
65+
EOL
66+
67+
run_test () {
68+
mkdir test_tmp/ && cd test_tmp/
69+
$1
70+
cd .. && rm -rf test_tmp/
71+
}
72+
73+
get_runtime_of_command () {
74+
TIMEFORMAT=%R
75+
76+
# runtime=$( { time ($1 &> /dev/null); } 2>&1 1>/dev/null)
77+
runtime=$( { time $1; } 2>&1 1>/dev/null)
78+
runtime=${runtime#+++ $1}
79+
runtime=$(python -c "print($runtime)")
80+
81+
echo $runtime
82+
}
83+
84+
# Define tests
85+
test_gpu_speed_mnist () {
86+
echo "Testing: MNIST, GPU"
87+
88+
git clone https://github.com/yf225/examples.git -b benchmark_test
89+
90+
cd examples/mnist
91+
92+
pip install -r requirements.txt
93+
94+
# Download data
95+
python main.py --epochs 0
96+
97+
SAMPLE_ARRAY=()
98+
NUM_RUNS=5
99+
100+
for (( i=1; i<=$NUM_RUNS; i++ )) do
101+
runtime=$(get_runtime_of_command "python main.py --epochs 1 --no-log")
102+
echo $runtime
103+
SAMPLE_ARRAY+=(${runtime})
104+
done
105+
106+
cd ../..
107+
108+
python ../compare_with_baseline.py ${FUNCNAME[0]} "${SAMPLE_ARRAY[@]}"
109+
}
110+
111+
test_gpu_speed_word_language_model () {
112+
echo "Testing: word language model on Wikitext-2, GPU"
113+
114+
git clone https://github.com/yf225/examples.git -b benchmark_test
115+
116+
cd examples/word_language_model
117+
118+
SAMPLE_ARRAY=()
119+
NUM_RUNS=5
120+
121+
for (( i=1; i<=$NUM_RUNS; i++ )) do
122+
runtime=$(get_runtime_of_command "python main.py --cuda --epochs 1")
123+
echo $runtime
124+
SAMPLE_ARRAY+=(${runtime})
125+
done
126+
127+
cd ../..
128+
129+
python ../compare_with_baseline.py ${FUNCNAME[0]} "${SAMPLE_ARRAY[@]}"
130+
}
131+
132+
test_gpu_speed_cudnn_lstm () {
133+
echo "Testing: CuDNN LSTM, GPU"
134+
135+
git clone https://github.com/yf225/benchmark.git
136+
137+
cd benchmark/scripts/
138+
139+
SAMPLE_ARRAY=()
140+
NUM_RUNS=5
141+
142+
for (( i=1; i<=$NUM_RUNS; i++ )) do
143+
runtime=$(get_runtime_of_command "python cudnn_lstm.py")
144+
echo $runtime
145+
SAMPLE_ARRAY+=(${runtime})
146+
done
147+
148+
cd ../..
149+
150+
python ../compare_with_baseline.py ${FUNCNAME[0]} "${SAMPLE_ARRAY[@]}"
151+
}
152+
153+
test_gpu_speed_lstm () {
154+
echo "Testing: LSTM, GPU"
155+
156+
git clone https://github.com/yf225/benchmark.git
157+
158+
cd benchmark/scripts/
159+
160+
SAMPLE_ARRAY=()
161+
NUM_RUNS=5
162+
163+
for (( i=1; i<=$NUM_RUNS; i++ )) do
164+
runtime=$(get_runtime_of_command "python lstm.py")
165+
echo $runtime
166+
SAMPLE_ARRAY+=(${runtime})
167+
done
168+
169+
cd ../..
170+
171+
python ../compare_with_baseline.py ${FUNCNAME[0]} "${SAMPLE_ARRAY[@]}"
172+
}
173+
174+
test_gpu_speed_mlstm () {
175+
echo "Testing: MLSTM, GPU"
176+
177+
git clone https://github.com/yf225/benchmark.git
178+
179+
cd benchmark/scripts/
180+
181+
SAMPLE_ARRAY=()
182+
NUM_RUNS=5
183+
184+
for (( i=1; i<=$NUM_RUNS; i++ )) do
185+
runtime=$(get_runtime_of_command "python mlstm.py")
186+
echo $runtime
187+
SAMPLE_ARRAY+=(${runtime})
188+
done
189+
190+
cd ../..
191+
192+
python ../compare_with_baseline.py ${FUNCNAME[0]} "${SAMPLE_ARRAY[@]}"
193+
}
194+
195+
echo "ENTERED_USER_LAND"
196+
197+
# Run tests
198+
run_test test_gpu_speed_mnist
199+
run_test test_gpu_speed_word_language_model
200+
run_test test_gpu_speed_cudnn_lstm
201+
run_test test_gpu_speed_lstm
202+
run_test test_gpu_speed_mlstm
203+
204+
echo "EXITED_USER_LAND"

0 commit comments

Comments
 (0)