@@ -236,6 +236,8 @@ test_dynamo_shard() {
236236 test_fx \
237237 test_package \
238238 test_legacy_vmap \
239+ functorch/test_dims \
240+ functorch/test_aotdispatch \
239241 --shard " $1 " " $NUM_TEST_SHARDS "
240242 --verbose
241243 assert_git_not_dirty
@@ -264,7 +266,7 @@ DYNAMO_BENCHMARK_FLAGS=()
264266
265267if [[ " ${TEST_CONFIG} " == * aot_eager* ]]; then
266268 DYNAMO_BENCHMARK_FLAGS+=(--backend aot_eager)
267- elif [[ " ${TEST_CONFIG} " == * inductor* ]]; then
269+ elif [[ " ${TEST_CONFIG} " == * inductor* && " ${TEST_CONFIG} " != * perf * ]]; then
268270 DYNAMO_BENCHMARK_FLAGS+=(--inductor)
269271fi
270272
@@ -278,6 +280,46 @@ else
278280 DYNAMO_BENCHMARK_FLAGS+=(--device cuda)
279281fi
280282
283+ test_perf_for_dashboard () {
284+ TEST_REPORTS_DIR=$( pwd)
285+ mkdir -p " $TEST_REPORTS_DIR "
286+
287+ local suite=" $1 "
288+ shift
289+
290+ for dtype in amp float32; do
291+ # Run accuracy test
292+ # All the accuracy tests can be skipped once the CI accuracy checking is stable enough
293+ for backend in eager aot_eager; do
294+ python " benchmarks/dynamo/$suite .py"
295+ --accuracy --" $dtype " " $backend " " $@ "
296+ --output " $TEST_REPORTS_DIR /${backend} _${suite} _${dtype} _training_cuda_accuracy.csv"
297+ done
298+
299+ # Run accuracy test for inductor with different configs
300+ # --disable-cudagraphs is the default inductor behavior
301+ # TODO: update here once cudagraphs is turned on as default
302+ backend=inductor
303+ python " benchmarks/dynamo/$suite .py"
304+ --accuracy --" $dtype " " $backend " " $@ "
305+ --output " $TEST_REPORTS_DIR /${backend} _no_cudagraphs_${suite} _${dtype} _training_cuda_accuracy.csv"
306+ python " benchmarks/dynamo/$suite .py"
307+ --accuracy --" $dtype " " $backend " " $@ "
308+ --output " $TEST_REPORTS_DIR /${backend} _with_cudagraphs_${suite} _${dtype} _training_cuda_accuracy.csv"
309+
310+ # Run performance test
311+ # Skip dynamo-eager and aot-eager for performance test
312+ # Run performance test for inductor with different configs
313+ # TODO: add more configs here, e.g. dynamic-shapes, max-autotune, etc.
314+ python " benchmarks/dynamo/$suite .py"
315+ --performance --cold-start-latency --" $dtype " " $backend " " $@ "
316+ --output " $TEST_REPORTS_DIR /${backend} _no_cudagraphs_${suite} _${dtype} _training_cuda_performance.csv"
317+ python " benchmarks/dynamo/$suite .py"
318+ --performance --cold-start-latency --" $dtype " " $backend " " $@ "
319+ --output " $TEST_REPORTS_DIR /${backend} _with_cudagraphs_${suite} _${dtype} _training_cuda_performance.csv"
320+ done
321+ }
322+
281323test_single_dynamo_benchmark () {
282324 # Usage: test_single_dynamo_benchmark inductor_inference huggingface 0 --args-for-script
283325
@@ -302,15 +344,12 @@ test_single_dynamo_benchmark() {
302344
303345 if [[ " ${TEST_CONFIG} " == * perf_compare* ]]; then
304346 python " benchmarks/dynamo/$suite .py"
305- --ci --performance --disable-cudagraphs \
306- " ${DYNAMO_BENCHMARK_FLAGS[@]} "
307- " $@ " " ${partition_flags[@]} "
347+ --ci --performance --disable-cudagraphs --inductor \
348+ " ${DYNAMO_BENCHMARK_FLAGS[@]} " " $@ " " ${partition_flags[@]} "
308349 --output " $TEST_REPORTS_DIR /${name} _${suite} .csv"
309350 elif [[ " ${TEST_CONFIG} " == * perf* ]]; then
310- # MKL_THREADING_LAYER=GNU to mitigate https://github.com/pytorch/pytorch/issues/37377
311- MKL_THREADING_LAYER=GNU python benchmarks/dynamo/runner.py --suites=" $suite "
312- --base-sha=" $BASE_SHA " " ${partition_flags[@]} "
313- --no-graphs --no-update-archive --no-gh-comment " $@ "
351+ test_perf_for_dashboard " $suite "
352+ " ${DYNAMO_BENCHMARK_FLAGS[@]} " " $@ " " ${partition_flags[@]} "
314353 else
315354 python " benchmarks/dynamo/$suite .py"
316355 --ci --accuracy --timing --explain \
@@ -322,6 +361,7 @@ test_single_dynamo_benchmark() {
322361 if [[ " ${TEST_CONFIG} " == * inductor* ]] && [[ " ${TEST_CONFIG} " != * cpu_accuracy* ]] && [[ " ${TEST_CONFIG} " != * dynamic* ]]; then
323362 # because I haven't dealt with dynamic expected artifacts yet,
324363 # and non-inductor jobs (e.g. periodic, cpu-accuracy) may have different set of expected models.
364+ # TODO: make update_expected.py produces combined expected csv file
325365 python benchmarks/dynamo/check_graph_breaks.py \
326366 --actual " $TEST_REPORTS_DIR /${name} _$suite .csv"
327367 --expected " benchmarks/dynamo/ci_expected_accuracy/${name} _${suite}${shard_id} .csv"
@@ -339,11 +379,10 @@ test_dynamo_benchmark() {
339379 shift
340380
341381 if [[ " ${TEST_CONFIG} " == * perf_compare* ]]; then
342- test_single_dynamo_benchmark " amp " " $suite " " $shard_id " " $@ "
382+ test_single_dynamo_benchmark " training " " $suite " " $shard_id " " $@ "
343383 elif [[ " ${TEST_CONFIG} " == * perf* ]]; then
344- # Performance test training only, for float32 and amp
345- test_single_dynamo_benchmark " amp" " $suite " " $shard_id " " $TEST_REPORTS_DIR " " $@ "
346- test_single_dynamo_benchmark " float32" " $suite " " $shard_id " " $TEST_REPORTS_DIR " " $@ "
384+ # Performance test training only
385+ test_single_dynamo_benchmark " training" " $suite " " $shard_id " " $@ "
347386 else
348387 # Check inference with --float32
349388 test_single_dynamo_benchmark " inference" " $suite " " $shard_id " " $@ "
@@ -532,6 +571,10 @@ test_vulkan() {
532571}
533572
534573test_distributed () {
574+ # Smuggle a few multi-gpu tests here so that we don't have to request another large node
575+ echo " Testing multi_gpu tests in test_torchinductor"
576+ pytest test/inductor/test_torchinductor.py -k test_multi_gpu
577+
535578 echo " Testing distributed python tests"
536579 time python test/run_test.py --distributed-tests --shard " $SHARD_NUMBER " " $NUM_TEST_SHARDS "
537580 assert_git_not_dirty
@@ -803,12 +846,6 @@ test_executorch() {
803846 assert_git_not_dirty
804847}
805848
806- # TODO: Include this in the Docker image
807- if [[ " ${TEST_CONFIG} " == * _perf* ]]; then
808- install_matplotlib
809- install_tabulate
810- fi
811-
812849if ! [[ " ${BUILD_ENVIRONMENT} " == * libtorch* || " ${BUILD_ENVIRONMENT} " == * -bazel-* || " ${BUILD_ENVIRONMENT} " == * -tsan* ]]; then
813850 (cd test && python -c " import torch; print(torch.__config__.show())"
814851 (cd test && python -c " import torch; print(torch.__config__.parallel_info())"
@@ -848,7 +885,8 @@ elif [[ "${TEST_CONFIG}" == *dynamo* && "${SHARD_NUMBER}" == 2 && $NUM_TEST_SHAR
848885elif [[ " ${TEST_CONFIG} " == * huggingface* ]]; then
849886 install_torchvision
850887 install_huggingface
851- test_dynamo_benchmark huggingface " "
888+ id=$(( SHARD_NUMBER- 1 ))
889+ test_dynamo_benchmark huggingface " $id "
852890elif [[ " ${TEST_CONFIG} " == * timm* ]]; then
853891 install_torchvision
854892 install_timm
@@ -862,12 +900,13 @@ elif [[ "${TEST_CONFIG}" == *torchbench* ]]; then
862900 fi
863901 install_torchtext
864902 install_torchvision
903+ id=$(( SHARD_NUMBER- 1 ))
865904 if [[ " ${TEST_CONFIG} " == * inductor_torchbench_smoketest_perf* ]]; then
866905 checkout_install_torchbench hf_Bert hf_Albert timm_efficientdet timm_vision_transformer
867906 PYTHONPATH=$( pwd)
868907 else
869908 checkout_install_torchbench
870- PYTHONPATH=$( pwd) " "
909+ PYTHONPATH=$( pwd) " $id "
871910 fi
872911elif [[ " ${TEST_CONFIG} " == * inductor* && " ${SHARD_NUMBER} " == 1 ]]; then
873912 install_torchvision
@@ -902,8 +941,6 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-tsan* ]]; then
902941 test_libtorch || true
903942elif [[ " ${TEST_CONFIG} " = docs_test ]]; then
904943 test_docs_test
905- elif [[ " ${TEST_CONFIG} " == * functorch* ]]; then
906- test_functorch
907944else
908945 install_torchvision
909946 install_monkeytype
0 commit comments