Commit 08a74a32 authored by Myle Ott's avatar Myle Ott
Browse files

Fix gradient clipping when --clip-norm=0

parent fd28c880

fairseq/multiprocessing_trainer.py

+3 −0
Changes for fairseq/multiprocessing_trainer.py: 3 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -265,7 +265,10 @@ class MultiprocessingTrainer(MultiprocessingEventLoop): 
        self._all_reduce_and_rescale_grads(grad_denom)



        # clip grads

        if self.args.clip_norm > 0:

            grad_norm = torch.nn.utils.clip_grad_norm(self.model.parameters(), self.args.clip_norm)

        else:

            grad_norm = math.sqrt(sum([p.grad.data.norm()**2 for p in self.model.parameters()]))



        # take an optimization step

        self.optimizer.step()