Skip to content

Commit efed242

Browse files
jbaczeknv-kkudrynski
jbaczek
authored and
nv-kkudrynski
committed
[Transformer/PyTorch] Add torch.cuda.synchronize() calls
1 parent 6a16011 commit efed242

4 files changed

Lines changed: 21 additions & 0 deletions

File tree

‎PyTorch/Translation/Transformer/fairseq/log_helper.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import dllogger
99
from dllogger import Backend, JSONStreamBackend
1010
from tensorboardX import SummaryWriter
11+
import torch
1112

1213

1314
class AverageMeter():
@@ -43,6 +44,7 @@ def __init__(self):
4344

4445
def reset(self):
4546
self.updated = False
47+
torch.cuda.synchronize()
4648
self.start = time.time()
4749
self.n = 0
4850

@@ -56,6 +58,7 @@ def value(self):
5658

5759
@property
5860
def elapsed_time(self):
61+
torch.cuda.synchronize()
5962
return time.time() - self.start
6063

6164

@@ -70,6 +73,7 @@ def __init__(self, verbosity, agg_dict):
7073
self.metrics.flushed = True
7174
self.step = 0
7275
self.epoch = 0
76+
torch.cuda.synchronize()
7377
self.start_time = time.time()
7478

7579
@property
@@ -115,6 +119,7 @@ def flush(self):
115119
result_string += _name + ' {:.3f} |'.format(agg.value)
116120
agg.reset()
117121

122+
torch.cuda.synchronize()
118123
result_string += 'walltime {:.3f} |'.format(time.time() - self.start_time)
119124
self.metrics.flushed = True
120125
print(result_string)

‎PyTorch/Translation/Transformer/fairseq/meters.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
# can be found in the PATENTS file in the same directory.
77

88
import time
9+
import torch
910

1011

1112
class AverageMeter(object):
@@ -33,12 +34,14 @@ def __init__(self, init=0):
3334

3435
def reset(self, init=0):
3536
self.init = init
37+
torch.cuda.synchronize()
3638
self.start = time.time()
3739
self.n = 0
3840
self.last_update = time.time()
3941

4042
def update(self, val=1):
4143
self.n += val
44+
torch.cuda.synchronize()
4245
self.last_update = time.time()
4346

4447
@property
@@ -47,6 +50,7 @@ def avg(self):
4750

4851
@property
4952
def elapsed_time(self):
53+
torch.cuda.synchronize()
5054
return self.init + (time.time() - self.start)
5155

5256
@property
@@ -61,9 +65,11 @@ def __init__(self):
6165
self.intervals = []
6266

6367
def start(self):
68+
torch.cuda.synchronize()
6469
self.start_time = time.time()
6570

6671
def stop(self, n=1):
72+
torch.cuda.synchronize()
6773
if self.start_time is not None:
6874
delta = time.time() - self.start_time
6975
self.intervals.append(delta)

‎PyTorch/Translation/Transformer/inference.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ def main(args):
151151

152152
use_cuda = torch.cuda.is_available() and not args.cpu
153153

154+
torch.cuda.synchronize()
154155
processing_start = time.time()
155156

156157
# Load ensemble
@@ -229,14 +230,18 @@ def process_batch(batch):
229230
tokens = tokens.cuda()
230231
lengths = lengths.cuda()
231232

233+
torch.cuda.synchronize()
232234
translation_start = time.time()
235+
233236
gen_timer.start()
234237
translations = translator.generate(
235238
tokens,
236239
lengths,
237240
maxlen=int(args.max_len_a * tokens.size(1) + args.max_len_b),
238241
)
239242
gen_timer.stop(sum(len(h[0]['tokens']) for h in translations))
243+
244+
torch.cuda.synchronize()
240245
dllogger.log(step='infer', data={'latency': time.time() - translation_start})
241246

242247
return [make_result(batch.srcs[i], t) for i, t in enumerate(translations)]
@@ -262,6 +267,7 @@ def process_batch(batch):
262267
if args.file:
263268
data_descriptor.close()
264269

270+
torch.cuda.synchronize()
265271
log_dict = {
266272
'throughput': 1./gen_timer.avg,
267273
'latency_avg': sum(gen_timer.intervals)/len(gen_timer.intervals),

‎PyTorch/Translation/Transformer/train.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ def train(args, trainer, epoch_itr):
164164

165165
max_update = args.max_update or math.inf
166166
num_batches = len(epoch_itr)
167+
torch.cuda.synchronize()
167168
begin = time.time()
168169

169170
# reset meters
@@ -189,6 +190,7 @@ def train(args, trainer, epoch_itr):
189190
if trainer.get_num_updates() >= max_update:
190191
break
191192

193+
torch.cuda.synchronize()
192194
print('Epoch time:', time.time() - begin)
193195

194196
# Print epoch stats and reset training meters
@@ -235,6 +237,7 @@ def validate(args, trainer, datasets, subsets):
235237

236238
def score(args, trainer, dataset, src_dict, tgt_dict, ref_file):
237239

240+
torch.cuda.synchronize()
238241
begin = time.time()
239242

240243
src_dict = deepcopy(src_dict) # This is necessary, generation of translations
@@ -324,6 +327,7 @@ def score(args, trainer, dataset, src_dict, tgt_dict, ref_file):
324327
float(args.distributed_world_size)/gen_timer.avg
325328
))
326329

330+
torch.cuda.synchronize()
327331
print('| Eval completed in: {:.2f}s | {}CASED BLEU {:.2f}'.format(
328332
time.time()-begin,
329333
'' if args.test_cased_bleu else 'UN',

0 commit comments

Comments
 (0)