Skip to content

Commit e44f653

Browse files
author
Zafar Takhirov
committed
Update on "Adding Scalar add/mul."
Note: This should be landed ONLY after #24259 Differential Revision: [D16846006](https://our.internmc.facebook.com/intern/diff/D16846006)
2 parents 4f182f9 + d4c1606 commit e44f653

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1220
-216
lines changed

‎.circleci/cimodel/data/binary_build_data.py‎

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -59,20 +59,21 @@ def get_processor_arch_name(cuda_version):
5959
)),
6060
)
6161

62-
63-
# Why is this an option?
64-
# All the nightlies used to be devtoolset3 and built with the old gcc ABI. We
65-
# added a devtoolset7 option so that we could build nightlies with the new gcc
66-
# ABI. That didn't work since devtoolset7 can't build with the new gcc ABI. But
67-
# then we set devtoolset7 to be the default anyways, since devtoolset7
68-
# understands avx512, which is needed for good fbgemm performance.
69-
# This should be removed. The base dockers should just be upgraded to
70-
# devtoolset7 so we don't have to reinstall this in every build job.
71-
# The same machinery that this uses, though, should be retooled for a different
72-
# compiler toolchain that can build with the new gcc ABI.
73-
DEVTOOLSET_VERSIONS = [
74-
7,
75-
]
62+
# GCC config variants:
63+
#
64+
# All the nightlies (except libtorch with new gcc ABI) are built with devtoolset7,
65+
# which can only build with old gcc ABI. It is better than devtoolset3
66+
# because it understands avx512, which is needed for good fbgemm performance.
67+
#
68+
# Libtorch with new gcc ABI is built with gcc 5.4 on Ubuntu 16.04.
69+
LINUX_GCC_CONFIG_VARIANTS = OrderedDict(
70+
manywheel=['devtoolset7'],
71+
conda=['devtoolset7'],
72+
libtorch=[
73+
"devtoolset7",
74+
"gcc5.4_cxx11-abi",
75+
],
76+
)
7677

7778

7879
class TopLevelNode(ConfigNode):
@@ -95,13 +96,7 @@ def __init__(self, parent, os_name, cuda_versions, py_tree):
9596
self.props["cuda_versions"] = cuda_versions
9697

9798
def get_children(self):
98-
packaging_variants = [PackageFormatConfigNode(self, k, v) for k, v in self.py_tree.items()]
99-
100-
if self.find_prop("smoke"):
101-
filtered_packaging_variants = list(filter(lambda x: x.get_label() != "libtorch", packaging_variants))
102-
return filtered_packaging_variants
103-
else:
104-
return packaging_variants
99+
return [PackageFormatConfigNode(self, k, v) for k, v in self.py_tree.items()]
105100

106101

107102
class PackageFormatConfigNode(ConfigNode):
@@ -113,23 +108,23 @@ def __init__(self, parent, package_format, python_versions):
113108

114109
def get_children(self):
115110
if self.find_prop("os_name") == "linux":
116-
return [LinuxGccConfigNode(self, v) for v in DEVTOOLSET_VERSIONS]
111+
return [LinuxGccConfigNode(self, v) for v in LINUX_GCC_CONFIG_VARIANTS[self.find_prop("package_format")]]
117112
else:
118113
return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
119114

120115

121116
class LinuxGccConfigNode(ConfigNode):
122-
def __init__(self, parent, devtoolset_version):
123-
super(LinuxGccConfigNode, self).__init__(parent, "DEVTOOLSET=" + str(devtoolset_version))
117+
def __init__(self, parent, gcc_config_variant):
118+
super(LinuxGccConfigNode, self).__init__(parent, "GCC_CONFIG_VARIANT=" + str(gcc_config_variant))
124119

125-
self.props["devtoolset_version"] = devtoolset_version
120+
self.props["gcc_config_variant"] = gcc_config_variant
126121

127122
def get_children(self):
128123
cuda_versions = self.find_prop("cuda_versions")
129124

130125
# XXX devtoolset7 on CUDA 9.0 is temporarily disabled
131126
# see https://github.com/pytorch/pytorch/issues/20066
132-
if self.find_prop("devtoolset_version") == 7:
127+
if self.find_prop("gcc_config_variant") == 'devtoolset7':
133128
cuda_versions = filter(lambda x: x != "90", cuda_versions)
134129

135130
return [ArchConfigNode(self, v) for v in cuda_versions]

‎.circleci/cimodel/data/binary_build_definitions.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,25 @@
99

1010

1111
class Conf(object):
12-
def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, devtoolset_version):
12+
def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant):
1313

1414
self.os = os
1515
self.cuda_version = cuda_version
1616
self.pydistro = pydistro
1717
self.parms = parms
1818
self.smoke = smoke
1919
self.libtorch_variant = libtorch_variant
20-
self.devtoolset_version = devtoolset_version
20+
self.gcc_config_variant = gcc_config_variant
2121

2222
def gen_build_env_parms(self):
2323
elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
24-
if self.devtoolset_version is not None:
25-
elems.append("devtoolset" + str(self.devtoolset_version))
24+
if self.gcc_config_variant is not None:
25+
elems.append(str(self.gcc_config_variant))
2626
return elems
2727

2828
def gen_docker_image(self):
29+
if self.gcc_config_variant == 'gcc5.4_cxx11-abi':
30+
return miniutils.quote("yf225/pytorch-binary-docker-image-ubuntu16.04:latest")
2931

3032
docker_word_substitution = {
3133
"manywheel": "manylinux",
@@ -108,15 +110,15 @@ def gen_build_env_list(smoke):
108110
[c.find_prop("pyver")],
109111
c.find_prop("smoke"),
110112
c.find_prop("libtorch_variant"),
111-
c.find_prop("devtoolset_version"),
113+
c.find_prop("gcc_config_variant"),
112114
)
113115
newlist.append(conf)
114116

115117
return newlist
116118

117119

118120
def predicate_exclude_nonlinux_and_libtorch(config):
119-
return config.os == "linux" and (config.smoke or config.pydistro != "libtorch")
121+
return config.os == "linux"
120122

121123

122124
def add_build_entries(jobs_dict, phase, smoke, filter_predicate=lambda x: True):

0 commit comments

Comments
 (0)