Skip to content

Commit 4f1c381

Browse files
author
Meghan Lele
committed
Update on "[JIT] Add support for backend-lowered submodules"
**Summary** This commit adds support for using `Modules` that have been lowered as submodules in `ScriptModules`. **Test Plan** This commit adds execution and save/load tests to test_backends.py for backend-lowered submodules. **Fixes** This commit fixes #40069. [ghstack-poisoned]
2 parents 41c6aff + 6bb8779 commit 4f1c381

File tree

143 files changed

+9334
-4670
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

143 files changed

+9334
-4670
lines changed

‎.circleci/config.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -580,6 +580,7 @@ jobs:
580580
# See https://circleci.com/docs/2.0/env-vars/#using-parameters-and-bash-environment
581581
echo "export PARALLEL_FLAGS=\"${PARALLEL_FLAGS}\"" >> $BASH_ENV
582582
echo "export id=$id" >> $BASH_ENV
583+
echo "export COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}" >> $BASH_ENV
583584
- run:
584585
name: Check for no AVX instruction by default
585586
no_output_timeout: "20m"
@@ -596,8 +597,8 @@ jobs:
596597
}
597598
598599
if is_vanilla_build; then
599-
echo "apt-get update && apt-get install -y qemu-user" | docker exec -u root -i "$id" bash
600-
echo "cd workspace/build; qemu-x86_64 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU" | docker exec -u jenkins -i "$id" bash
600+
echo "apt-get update && apt-get install -y qemu-user gdb" | docker exec -u root -i "$id" bash
601+
echo "cd workspace/build; qemu-x86_64 -g 2345 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU & gdb ./bin/basic -ex 'set pagination off' -ex 'target remote :2345' -ex 'continue' -ex 'bt' -ex='set confirm off' -ex 'quit \$_isvoid(\$_exitcode)'" | docker exec -u jenkins -i "$id" bash
601602
else
602603
echo "Skipping for ${BUILD_ENVIRONMENT}"
603604
fi
@@ -622,6 +623,12 @@ jobs:
622623
echo "cd workspace; python test/print_test_stats.py test" | docker exec -u jenkins -i "$id" bash
623624
echo "Retrieving test reports"
624625
docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!'
626+
echo "Stopping current container"
627+
docker container stop $id
628+
echo "Pruning stopped containers"
629+
docker container prune -f
630+
echo "Deleting the image"
631+
docker image rm ${COMMIT_DOCKER_IMAGE}
625632
when: always
626633
- store_test_results:
627634
path: test-reports
@@ -1943,6 +1950,7 @@ jobs:
19431950
update_s3_htmls: &update_s3_htmls
19441951
machine:
19451952
image: ubuntu-1604:201903-01
1953+
resource_class: medium
19461954
steps:
19471955
- checkout
19481956
- setup_linux_system_environment

‎.circleci/verbatim-sources/job-specs/binary_update_htmls.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
update_s3_htmls: &update_s3_htmls
1010
machine:
1111
image: ubuntu-1604:201903-01
12+
resource_class: medium
1213
steps:
1314
- checkout
1415
- setup_linux_system_environment

‎.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ jobs:
118118
# See https://circleci.com/docs/2.0/env-vars/#using-parameters-and-bash-environment
119119
echo "export PARALLEL_FLAGS=\"${PARALLEL_FLAGS}\"" >> $BASH_ENV
120120
echo "export id=$id" >> $BASH_ENV
121+
echo "export COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}" >> $BASH_ENV
121122
- run:
122123
name: Check for no AVX instruction by default
123124
no_output_timeout: "20m"
@@ -134,8 +135,8 @@ jobs:
134135
}
135136
136137
if is_vanilla_build; then
137-
echo "apt-get update && apt-get install -y qemu-user" | docker exec -u root -i "$id" bash
138-
echo "cd workspace/build; qemu-x86_64 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU" | docker exec -u jenkins -i "$id" bash
138+
echo "apt-get update && apt-get install -y qemu-user gdb" | docker exec -u root -i "$id" bash
139+
echo "cd workspace/build; qemu-x86_64 -g 2345 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU & gdb ./bin/basic -ex 'set pagination off' -ex 'target remote :2345' -ex 'continue' -ex 'bt' -ex='set confirm off' -ex 'quit \$_isvoid(\$_exitcode)'" | docker exec -u jenkins -i "$id" bash
139140
else
140141
echo "Skipping for ${BUILD_ENVIRONMENT}"
141142
fi
@@ -160,6 +161,12 @@ jobs:
160161
echo "cd workspace; python test/print_test_stats.py test" | docker exec -u jenkins -i "$id" bash
161162
echo "Retrieving test reports"
162163
docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!'
164+
echo "Stopping current container"
165+
docker container stop $id
166+
echo "Pruning stopped containers"
167+
docker container prune -f
168+
echo "Deleting the image"
169+
docker image rm ${COMMIT_DOCKER_IMAGE}
163170
when: always
164171
- store_test_results:
165172
path: test-reports

‎.jenkins/pytorch/macos-test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ test_python_all() {
6363
# Increase default limit on open file handles from 256 to 1024
6464
ulimit -n 1024
6565

66-
python test/run_test.py --verbose --exclude test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --determine-from="$DETERMINE_FROM"
66+
python test/run_test.py --verbose --exclude test_jit_cuda_fuser_profiling test_jit_cuda_fuser_legacy test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --determine-from="$DETERMINE_FROM"
6767

6868
assert_git_not_dirty
6969
}

‎.jenkins/pytorch/test.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,17 +150,17 @@ test_python_nn() {
150150
}
151151

152152
test_python_ge_config_profiling() {
153-
time python test/run_test.py --include test_jit_profiling test_jit_fuser_te --verbose --determine-from="$DETERMINE_FROM"
153+
time python test/run_test.py --include test_jit_cuda_fuser_profiling test_jit_profiling test_jit_fuser_te --verbose --determine-from="$DETERMINE_FROM"
154154
assert_git_not_dirty
155155
}
156156

157157
test_python_ge_config_legacy() {
158-
time python test/run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="$DETERMINE_FROM"
158+
time python test/run_test.py --include test_jit_cuda_fuser_legacy test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="$DETERMINE_FROM"
159159
assert_git_not_dirty
160160
}
161161

162162
test_python_all_except_nn_and_cpp_extensions() {
163-
time python test/run_test.py --exclude test_nn test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --verbose --determine-from="$DETERMINE_FROM"
163+
time python test/run_test.py --exclude test_jit_cuda_fuser_profiling test_jit_cuda_fuser_legacy test_nn test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --verbose --determine-from="$DETERMINE_FROM"
164164
assert_git_not_dirty
165165
}
166166

‎.jenkins/pytorch/win-test-helpers/test_python_all_except_nn.bat‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
2-
cd test && python run_test.py --exclude test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --verbose --determine-from="%1" && cd ..
2+
cd test && python run_test.py --exclude test_jit_cuda_fuser_profiling test_jit_cuda_fuser_legacy test_jit_profiling test_jit_legacy test_jit_fuser_legacy test_jit_fuser_te test_tensorexpr --verbose --determine-from="%1" && cd ..
33
if ERRORLEVEL 1 exit /b 1

‎aten/src/ATen/Declarations.cwrap‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -146,19 +146,6 @@
146146
- THIndexTensor* index
147147
- real value
148148
]]
149-
[[
150-
name: _th_equal
151-
cname: equal
152-
backends:
153-
- CUDA
154-
cuda_bool: True
155-
variants:
156-
- function
157-
return: bool
158-
arguments:
159-
- THTensor* self
160-
- THTensor* other
161-
]]
162149
[[
163150
name: _th_mode
164151
variants: function

‎aten/src/ATen/core/interned_strings.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ namespace c10 {
188188
_(prim, unchecked_unwrap_optional) \
189189
_(aten, __contains__) \
190190
_(prim, BailoutTemplate) \
191+
_(prim, grad) \
191192
_(aten, zero_) \
192193
_(aten, fill_) \
193194
FORALL_ATEN_BASE_SYMBOLS(_) \

‎aten/src/ATen/core/jit_type.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1481,7 +1481,7 @@ inline TypePtr TensorType::fromBoolType() {
14811481

14821482
inline c10::optional<c10::ScalarType> tryScalarTypeFromJitType(const c10::TypePtr & type) {
14831483
if (type == FloatType::get()) {
1484-
return at::ScalarType::Double;
1484+
return at::typeMetaToScalarType(c10::get_default_dtype());
14851485
} else if (type == IntType::get()) {
14861486
return at::ScalarType::Long;
14871487
} else if (type == BoolType::get()) {

‎aten/src/ATen/native/RNN.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
#include <ATen/native/quantized/cpu/qnnpack_utils.h>
1010
#include <torch/custom_class.h>
1111

12-
torch::jit::class_<LinearPackedParamsBase> register_linear_params();
12+
torch::class_<LinearPackedParamsBase> register_linear_params();
1313

1414
namespace at { namespace native {
1515

0 commit comments

Comments
 (0)