Fixes #{issue number}

It's not clear by illustrating 'gradients from each node are averaged' in the documentation of DistributedDataParallel. Many people, including me, have a totally wrong understanding on this part. I add a note into the documentation to make it more straight forward and more user friendly.

Here is some toy code to illustrate my point:

• non-DistributedDataParallel version

  import torch
  import torch.nn as nn
  
  x = torch.tensor([-1, 2, -3, 4], dtype=torch.float).view(-1, 1)
  print("input:", x)
  
  model = nn.Linear(in_features=1, out_features=1, bias=False)
  model.weight.data.zero_()
  model.weight.data.add_(1.0)
  
  opti = torch.optim.SGD(model.parameters(), lr=0.001)
  opti.zero_grad()
  
  y = model(x)
  
  label = torch.zeros(4, 1, dtype=torch.float)
  loss = torch.sum((y - label)**2)
  
  loss.backward()
  opti.step()
  
  print("grad:", model.weight.grad)
  print("updated weight:\n", model.weight)
  
  # OUTPUT
  # $ python test.py
  # input: tensor([[-1.],
  #         [ 2.],
  #         [-3.],
  #         [ 4.]])
  # grad: tensor([[60.]])
  # updated weight:
  #  Parameter containing:
  # tensor([[0.9400]], requires_grad=True)

• DistributedDataParallel version

  import os
  import torch
  import torch.nn as nn
  import torch.distributed as dist
  from torch.multiprocessing import Process
  
  def run(rank, size):
      x = torch.tensor([-(1 + 2 * rank), 2 + 2 * rank], dtype=torch.float).view(-1, 1)
      print("input:", x)
  
      model = nn.Linear(in_features=1, out_features=1, bias=False)
      model.weight.data.zero_()
      model.weight.data.add_(1.0)
      model = torch.nn.parallel.DistributedDataParallel(model)
  
      opti = torch.optim.SGD(model.parameters(), lr=0.001)
      opti.zero_grad()
  
      y = model(x)
  
      label = torch.zeros(2, 1, dtype=torch.float)
      loss = torch.sum((y.view(-1, 1) - label)**2)
  
      loss.backward()
      opti.step()
  
      if rank == 0:
          print("grad:", model.module.weight.grad)
          print("updated weight:\n", model.module.weight)
  
  
  def init_process(rank, size, fn, backend="gloo"):
      os.environ['MASTER_ADDR'] = '127.0.0.1'
      os.environ['MASTER_PORT'] = '29500'
      dist.init_process_group(backend, rank=rank, world_size=size)
      fn(rank, size)
  
  
  if __name__ == "__main__":
      size = 2
      process = []
      for rank in range(size):
          p = Process(target=init_process, args=(rank, size, run))
          p.start()
          process.append(p)
  
      for p in process:
          p.join()
  
  # OUTPUT
  # $ python test_d.py
  # input: tensor([[-3.],
  #         [ 4.]])input: tensor([[-1.],
  #         [ 2.]])
  
  # grad: tensor([[30.]])
  # updated weight:
  #  Parameter containing:
  # tensor([[0.9700]], requires_grad=True)