Remove CUDNN_BATCHNORM_SPATIAL_PERSISTENT mode for accuracy (#13844)

xw285cornell · facebook-github-bot · commit 561bc0902640 · 2018-11-13T11:17:16.000-08:00
Summary: Pull Request resolved: #13844 In S163230, we've found CuDNN 7 upgrade causes accuracy drop in training convolution network such as ResNeXt-101 (~0% accuracy), and video R(2+1)D (65 --> 63%). We've fixed this in Caffe2 D9601217, and we should do the same to ATen as well. Reviewed By: ezyang Differential Revision: D13025486 fbshipit-source-id: 04f4f0d9af6287b0400ca1842fb2cdac1f8cdb70
diff --git a/aten/src/ATen/native/cudnn/BatchNorm.cpp b/aten/src/ATen/native/cudnn/BatchNorm.cpp
@@ -88,10 +88,10 @@ std::tuple<Tensor, Tensor, Tensor> cudnn_batch_norm(
     mode = CUDNN_BATCHNORM_PER_ACTIVATION;
   } else {
     mode = CUDNN_BATCHNORM_SPATIAL;
-#if CUDNN_VERSION >= 7003
-    if(training)
-      mode = CUDNN_BATCHNORM_SPATIAL_PERSISTENT;
-#endif
+    // TODO: The new CUDNN_BATCHNORM_SPATIAL_PERSISTENT mode was
+    // introduced in CuDNN 7 for performance optimization, but it results in
+    // accuracy losses in convolution models such as ResNeXt-101 and
+    // video R(2+1)D. We will fall back to the normal CUDNN_BATCHNORM_SPATIAL
   }
 
   auto output_t = at::empty(input->sizes(), input->options());
@@ -183,11 +183,11 @@ std::tuple<Tensor, Tensor, Tensor> cudnn_batch_norm_backward(
   if (input->dim() == 2) {
     mode = CUDNN_BATCHNORM_PER_ACTIVATION;
   } else {
-#if CUDNN_VERSION >= 7003
-    mode = CUDNN_BATCHNORM_SPATIAL_PERSISTENT;
-#else
+    // TODO: The new CUDNN_BATCHNORM_SPATIAL_PERSISTENT mode was
+    // introduced in CuDNN 7 for performance optimization, but it results in
+    // accuracy losses in convolution models such as ResNeXt-101 and
+    // video R(2+1)D. We will fall back to the normal CUDNN_BATCHNORM_SPATIAL
     mode = CUDNN_BATCHNORM_SPATIAL;
-#endif
   }
 
   auto grad_input_t  = at::empty(input->sizes(), input->options());
