pytorch · InnovArul · Jun 20, 2019 · Jun 20, 2019 · Jun 20, 2019 · Jun 20, 2019
diff --git a/.circleci/README.md b/.circleci/README.md
@@ -1,3 +1,23 @@
+Structure of CI
+===============
+
+setup job:
+1. Does a git checkout
+2. Persists CircleCI scripts (everything in `.circleci`) into a workspace.  Why?
+   We don't always do a Git checkout on all subjobs, but we usually
+   still want to be able to call scripts one way or another in a subjob.
+   Persisting files this way lets us have access to them without doing a
+   checkout.  This workspace is conventionally mounted on `~/workspace`
+   (this is distinguished from `~/project`, which is the conventional
+   working directory that CircleCI will default to starting your jobs
+   in.)
+3. Write out the commit message to `.circleci/COMMIT_MSG`.  This is so
+   we can determine in subjobs if we should actually run the jobs or
+   not, even if there isn't a Git checkout.
+
+
+
+
 CircleCI configuration generator
 ================================
 
@@ -35,4 +55,4 @@ Future direction
 See comment [here](https://github.com/pytorch/pytorch/pull/17323#pullrequestreview-206945747):
 
 In contrast with a full recursive tree traversal of configuration dimensions,
-> in the future future I think we actually want to decrease our matrix somewhat and have only a few mostly-orthogonal builds that taste as many different features as possible on PRs, plus a more complete suite on every PR and maybe an almost full suite nightly/weekly (we don't have this yet). Specifying PR jobs in the future might be easier to read with an explicit list when we come to this.
+> in the future future I think we actually want to decrease our matrix somewhat and have only a few mostly-orthogonal builds that taste as many different features as possible on PRs, plus a more complete suite on every PR and maybe an almost full suite nightly/weekly (we don't have this yet). Specifying PR jobs in the future might be easier to read with an explicit list when we come to this.
diff --git a/.circleci/cimodel/data/binary_build_data.py b/.circleci/cimodel/data/binary_build_data.py
@@ -42,7 +42,7 @@ def get_processor_arch_name(cuda_version):
         "3.6m",
         "3.7m",
     ],
-    conda=dimensions.STANDARD_PYTHON_VERSIONS,
+    conda=dimensions.CONDA_PYTHON_VERSIONS,
     libtorch=[
         "2.7m",
     ],
@@ -52,7 +52,7 @@ def get_processor_arch_name(cuda_version):
     linux=(dimensions.CUDA_VERSIONS, LINUX_PACKAGE_VARIANTS),
     macos=([None], OrderedDict(
         wheel=dimensions.STANDARD_PYTHON_VERSIONS,
-        conda=dimensions.STANDARD_PYTHON_VERSIONS,
+        conda=dimensions.CONDA_PYTHON_VERSIONS,
         libtorch=[
             "2.7",
         ],
@@ -62,7 +62,6 @@ def get_processor_arch_name(cuda_version):
 
 DEVTOOLSET_VERSIONS = [
     3,
-    7,
 ]
 
 
@@ -86,7 +85,13 @@ def __init__(self, parent, os_name, cuda_versions, py_tree):
         self.props["cuda_versions"] = cuda_versions
 
     def get_children(self):
-        return [PackageFormatConfigNode(self, k, v) for k, v in self.py_tree.items()]
+        packaging_variants = [PackageFormatConfigNode(self, k, v) for k, v in self.py_tree.items()]
+
+        if self.find_prop("smoke"):
+            filtered_packaging_variants = list(filter(lambda x: x.get_label() != "libtorch", packaging_variants))
+            return filtered_packaging_variants
+        else:
+            return packaging_variants
 
 
 class PackageFormatConfigNode(ConfigNode):

diff --git a/.circleci/cimodel/data/caffe2_build_data.py b/.circleci/cimodel/data/caffe2_build_data.py
@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
 
-from cimodel.lib.conf_tree import ConfigNode, X
+from cimodel.lib.conf_tree import ConfigNode, X, XImportant
 from cimodel.lib.conf_tree import Ver
-import cimodel.data.dimensions as dimensions
 
 
 CONFIG_TREE_DATA = [
@@ -14,16 +13,17 @@
         (Ver("cuda", "9.0"), [
             # TODO make explicit that this is a "secret TensorRT build"
             #  (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749)
+            # TODO Uh oh, were we supposed to make this one important?!
             X("py2"),
-            X("cmake"),
+            XImportant("cmake"),
         ]),
-        (Ver("cuda", "9.1"), [X("py2")]),
-        (Ver("mkl"), [X("py2")]),
-        (Ver("gcc", "5"), [X("onnx_py2")]),
+        (Ver("cuda", "9.1"), [XImportant("py2")]),
+        (Ver("mkl"), [XImportant("py2")]),
+        (Ver("gcc", "5"), [XImportant("onnx_py2")]),
         (Ver("clang", "3.8"), [X("py2")]),
         (Ver("clang", "3.9"), [X("py2")]),
-        (Ver("clang", "7"), [X("py2")]),
-        (Ver("android"), [X("py2")]),
+        (Ver("clang", "7"), [XImportant("py2"), XImportant("onnx_py3.6")]),
+        (Ver("android"), [XImportant("py2")]),
     ]),
     (Ver("centos", "7"), [
         (Ver("cuda", "9.0"), [X("py2")]),
@@ -32,7 +32,7 @@
         # TODO ios and system aren't related. system qualifies where the python comes
         #  from (use the system python instead of homebrew or anaconda)
         (Ver("ios"), [X("py2")]),
-        (Ver("system"), [X("py2")]),
+        (Ver("system"), [XImportant("py2")]),
     ]),
 ]
 
@@ -54,6 +54,8 @@ def get_children(self):
         return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
 
     def is_build_only(self):
+        if str(self.find_prop("language_version")) == "onnx_py3.6":
+            return False
         return str(self.find_prop("compiler_version")) in [
             "gcc4.9",
             "clang3.8",
@@ -95,16 +97,13 @@ def init2(self, node_name):
         self.props["language_version"] = node_name
         self.props["build_only"] = self.is_build_only()
 
-    def get_children(self):
-
-        children = []
-        for phase in dimensions.PHASES:
-            if phase == "build" or not self.props["build_only"]:
-                children.append(PhaseConfigNode(self, phase, []))
-
-        return children
+    def child_constructor(self):
+        return ImportantConfigNode
 
 
-class PhaseConfigNode(TreeConfigNode):
+class ImportantConfigNode(TreeConfigNode):
     def init2(self, node_name):
-        self.props["phase_name"] = node_name
+        self.props["important"] = True
+
+    def get_children(self):
+        return []
diff --git a/.circleci/cimodel/data/caffe2_build_definitions.py b/.circleci/cimodel/data/caffe2_build_definitions.py
@@ -2,30 +2,35 @@
 
 from collections import OrderedDict
 
+import cimodel.data.dimensions as dimensions
 import cimodel.lib.conf_tree as conf_tree
+from cimodel.lib.conf_tree import Ver
 import cimodel.lib.miniutils as miniutils
 import cimodel.lib.visualization as visualization
 from cimodel.data.caffe2_build_data import CONFIG_TREE_DATA, TopLevelNode
 
 
-DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
+from dataclasses import dataclass
+
 
-DOCKER_IMAGE_VERSION = 276
+DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
 
+DOCKER_IMAGE_VERSION = 287
 
-class Conf(object):
-    def __init__(self, language, distro, compiler, phase, build_only):
 
-        self.language = language
-        self.distro = distro
-        self.compiler = compiler
-        self.phase = phase
-        self.build_only = build_only
+@dataclass
+class Conf:
+    language: str
+    distro: Ver
+    compiler: Ver
+    build_only: bool
+    is_important: bool
 
     # TODO: Eventually we can probably just remove the cudnn7 everywhere.
     def get_cudnn_insertion(self):
 
         omit = self.language == "onnx_py2" \
+            or self.language == "onnx_py3.6" \
             or self.compiler.name in ["android", "mkl", "clang"] \
             or str(self.distro) in ["ubuntu14.04", "macos10.13"]
 
@@ -44,9 +49,6 @@ def construct_phase_name(self, phase):
         root_parts = self.get_build_name_root_parts()
         return "_".join(root_parts + [phase]).replace(".", "_")
 
-    def get_name(self):
-        return self.construct_phase_name(self.phase)
-
     def get_platform(self):
         platform = self.distro.name
         if self.distro.name != "macos":
@@ -57,27 +59,29 @@ def gen_docker_image(self):
 
         lang_substitutions = {
             "onnx_py2": "py2",
+            "onnx_py3.6": "py3.6",
             "cmake": "py2",
         }
 
         lang = miniutils.override(self.language, lang_substitutions)
         parts = [lang] + self.get_build_name_middle_parts()
         return miniutils.quote(DOCKER_IMAGE_PATH_BASE + "-".join(parts) + ":" + str(DOCKER_IMAGE_VERSION))
 
-    def gen_yaml_tree(self):
+    def gen_yaml_tree(self, phase):
 
         tuples = []
 
         lang_substitutions = {
             "onnx_py2": "onnx-py2",
+            "onnx_py3.6": "onnx-py3.6",
         }
 
         lang = miniutils.override(self.language, lang_substitutions)
 
         parts = [
             "caffe2",
             lang,
-        ] + self.get_build_name_middle_parts() + [self.phase]
+        ] + self.get_build_name_middle_parts() + [phase]
 
         build_env = "-".join(parts)
         if not self.distro.name == "macos":
@@ -88,7 +92,7 @@ def gen_yaml_tree(self):
         if self.compiler.name == "ios":
             tuples.append(("BUILD_IOS", miniutils.quote("1")))
 
-        if self.phase == "test":
+        if phase == "test":
             # TODO cuda should not be considered a compiler
             if self.compiler.name == "cuda":
                 tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
@@ -103,11 +107,11 @@ def gen_yaml_tree(self):
 
         d = OrderedDict({"environment": OrderedDict(tuples)})
 
-        if self.phase == "test":
+        if phase == "test":
             resource_class = "large" if self.compiler.name != "cuda" else "gpu.medium"
             d["resource_class"] = resource_class
 
-        d["<<"] = "*" + "_".join(["caffe2", self.get_platform(), self.phase, "defaults"])
+        d["<<"] = "*" + "_".join(["caffe2", self.get_platform(), phase, "defaults"])
 
         return d
 
@@ -125,11 +129,11 @@ def instantiate_configs():
     for fc in found_configs:
 
         c = Conf(
-            fc.find_prop("language_version"),
-            fc.find_prop("distro_version"),
-            fc.find_prop("compiler_version"),
-            fc.find_prop("phase_name"),
-            fc.find_prop("build_only"),
+            language=fc.find_prop("language_version"),
+            distro=fc.find_prop("distro_version"),
+            compiler=fc.find_prop("compiler_version"),
+            build_only=fc.find_prop("build_only"),
+            is_important=fc.find_prop("important"),
         )
 
         config_list.append(c)
@@ -138,10 +142,13 @@ def instantiate_configs():
 
 
 def add_caffe2_builds(jobs_dict):
-
     configs = instantiate_configs()
     for conf_options in configs:
-        jobs_dict[conf_options.get_name()] = conf_options.gen_yaml_tree()
+        phases = ["build"]
+        if not conf_options.build_only:
+            phases = dimensions.PHASES
+        for phase in phases:
+            jobs_dict[conf_options.construct_phase_name(phase)] = conf_options.gen_yaml_tree(phase)
 
     graph = visualization.generate_graph(get_root())
     graph.draw("caffe2-config-dimensions.png", prog="twopi")
@@ -158,11 +165,23 @@ def get_caffe2_workflows():
     x = []
     for conf_options in filtered_configs:
 
-        requires = ["setup"]
+        phases = ["build"]
+        if not conf_options.build_only:
+            phases = dimensions.PHASES
+
+        for phase in phases:
+
+            requires = ["setup"]
+            sub_d = {"requires": requires}
+
+            if phase == "test":
+                requires.append(conf_options.construct_phase_name("build"))
 
-        if conf_options.phase == "test":
-            requires.append(conf_options.construct_phase_name("build"))
+            if not conf_options.is_important:
+                # If you update this, update
+                # pytorch_build_definitions.py too
+                sub_d["filters"] = {"branches": {"only": ["master", r"/ci-all\/.*/"]}}
 
-        x.append({conf_options.get_name(): {"requires": requires}})
+            x.append({conf_options.construct_phase_name(phase): sub_d})
 
     return x
diff --git a/.circleci/cimodel/data/dimensions.py b/.circleci/cimodel/data/dimensions.py
@@ -15,3 +15,9 @@
     "3.6",
     "3.7",
 ]
+
+CONDA_PYTHON_VERSIONS = [
+    "2.7",
+    "3.6",
+    "3.7",
+]
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,3 +15,9 @@ @@
         "3.6",
         "3.7",
     ]
+    CONDA_PYTHON_VERSIONS = [
+        "2.7",
+        "3.6",
+        "3.7",
+    ]