Don't run stubgen tests by default in runtests.py (python#8403)

JukkaL · web-flow · commit 35bdb07d656f · 2020-02-14T14:53:03.000Z
The tests are slow (around 56s sequentially on my laptop) and not very
likely to break changes not touching stubtest, so it seems reasonable
to make them opt-in outside of CI.
diff --git a/runtests.py b/runtests.py
@@ -28,27 +28,38 @@
 MYPYC_EXTERNAL = 'TestExternal'
 MYPYC_COMMAND_LINE = 'TestCommandLine'
 ERROR_STREAM = 'ErrorStreamSuite'
-
-
-ALL_NON_FAST = [CMDLINE,
-                SAMPLES,
-                TYPESHED,
-                PEP561,
-                EVALUATION,
-                DAEMON,
-                STUBGEN_CMD,
-                STUBGEN_PY,
-                MYPYC_RUN,
-                MYPYC_RUN_MULTI,
-                MYPYC_EXTERNAL,
-                MYPYC_COMMAND_LINE,
-                ERROR_STREAM]
+STUBTEST = 'StubtestUnit'
+STUBTEST_MISC = 'StubtestMiscUnit'
+STUBTEST_INTEGRATION = 'StubtestIntegration'
+
+
+ALL_NON_FAST = [
+    CMDLINE,
+    SAMPLES,
+    TYPESHED,
+    PEP561,
+    EVALUATION,
+    DAEMON,
+    STUBGEN_CMD,
+    STUBGEN_PY,
+    MYPYC_RUN,
+    MYPYC_RUN_MULTI,
+    MYPYC_EXTERNAL,
+    MYPYC_COMMAND_LINE,
+    ERROR_STREAM,
+    STUBTEST,
+    STUBTEST_MISC,
+    STUBTEST_INTEGRATION,
+]
 
 
 # These must be enabled by explicitly including 'mypyc-extra' on the command line.
 MYPYC_OPT_IN = [MYPYC_RUN,
                 MYPYC_RUN_MULTI]
 
+# These must be enabled by explicitly including 'stubtest' on the command line.
+STUBTEST_OPT_IN = [STUBTEST, STUBTEST_MISC, STUBTEST_INTEGRATION]
+
 # We split the pytest run into three parts to improve test
 # parallelization. Each run should have tests that each take a roughly similar
 # time to run.
@@ -76,6 +87,7 @@
     # Mypyc tests that aren't run by default, since they are slow and rarely
     # fail for commits that don't touch mypyc
     'mypyc-extra': 'pytest -k "%s"' % ' or '.join(MYPYC_OPT_IN),
+    'stubtest': 'pytest -k "%s"' % ' or '.join(STUBTEST_OPT_IN),
 }
 
 # Stop run immediately if these commands fail
