Give a better error when we run out of shared memory, instead of "RuntimeError: DataLoader worker (pid 13) is killed by signal: Bus error." #5040
Description
When I set num_workers=1 or other value greater than 0 in torch.utils.data.DataLoader, I get this error.
The detail of the error:
Traceback (most recent call last):
File "/opt/project/train.py", line 150, in <module>
dataset_sizes=dataset_sizes)
File "/opt/project/train.py", line 51, in train_model
outputs = model(inputs)
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/nn/modules/module.py", line 357, in __call__
result = self.forward(*input, **kwargs)
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 64, in forward
inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 75, in scatter
return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim)
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/nn/parallel/scatter_gather.py", line 30, in scatter_kwargs
inputs = scatter(inputs, target_gpus, dim) if inputs else []
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/nn/parallel/scatter_gather.py", line 25, in scatter
return scatter_map(inputs)
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/nn/parallel/scatter_gather.py", line 18, in scatter_map
return list(zip(*map(scatter_map, obj)))
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/nn/parallel/scatter_gather.py", line 15, in scatter_map
return Scatter.apply(target_gpus, None, dim, obj)
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/nn/parallel/_functions.py", line 68, in forward
outputs = comm.scatter(input, ctx.target_gpus, ctx.chunk_sizes, ctx.dim, streams)
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/cuda/comm.py", line 189, in scatter
outputs.append(chunk.cuda(device, async=True))
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/_utils.py", line 69, in _cuda
return new_type(self.size()).copy_(self, async)
File "/opt/conda/envs/pytorch-py3.6/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 172, in handler
_error_if_any_worker_fails()
RuntimeError: DataLoader worker (pid 13) is killed by signal: Bus error.