Commit d9f46c54 authored by Sergey Edunov's avatar Sergey Edunov
Browse files

Merge branch 'master' of github.com:facebookresearch/fairseq-py into prepare_wmt

parents 4185d3ed ee36a6f3

README.md

+2 −2
Changes for README.md: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -24,8 +24,8 @@ If you use the code in your paper, then please cite it as: 
* Python version 3.6

* A [PyTorch installation](http://pytorch.org/)



Currently fairseq-py requires installing PyTorch from source.

Please follow the instructions here: https://github.com/pytorch/pytorch#from-source.

Currently fairseq-py requires PyTorch version >= 0.3.0.

Please follow the instructions here: https://github.com/pytorch/pytorch#installation.



If you use Docker make sure to increase the shared memory size either with `--ipc=host` or `--shm-size` as command line

options to `nvidia-docker run`.

fairseq/clib/temporal_convolution_tbc/temporal_convolution_tbc.cpp

+1 −1
Changes for fairseq/clib/temporal_convolution_tbc/temporal_convolution_tbc.cpp: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -126,5 +126,5 @@ void TemporalConvolutionTBC_backward( 
  }



  auto tmp = dOutput.sum(0, false);

  dBias.assign_(tmp.sum(0));

  dBias.copy_(tmp.sum(0));

}

fairseq/criterions/cross_entropy.py

+13 −6
Changes for fairseq/criterions/cross_entropy.py: 13 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ class CrossEntropyCriterion(FairseqCriterion): 
    def __init__(self, args, dst_dict):

        super().__init__(args, dst_dict)



    def forward(self, model, sample):

    def forward(self, model, sample, reduce=True):

        """Compute the loss for the given sample.



        Returns a tuple with three elements:
@@ -26,12 +26,14 @@ class CrossEntropyCriterion(FairseqCriterion): 
        3) logging outputs to display while training

        """

        net_output = model(**sample['net_input'])

        input = net_output.view(-1, net_output.size(-1))

        lprobs = model.get_normalized_probs(net_output, log_probs=True)

        target = sample['target'].view(-1)

        loss = F.cross_entropy(input, target, size_average=False, ignore_index=self.padding_idx)

        loss = F.nll_loss(lprobs, target, size_average=False, ignore_index=self.padding_idx,

                          reduce=reduce)

        sample_size = sample['target'].size(0) if self.args.sentence_avg else sample['ntokens']

        logging_output = {

            'loss': loss.data[0],

            'loss': loss.data[0] if reduce else loss.data,

            'ntokens': sample['ntokens'],

            'sample_size': sample_size,

        }

        return loss, sample_size, logging_output
@@ -39,7 +41,12 @@ class CrossEntropyCriterion(FairseqCriterion): 
    @staticmethod

    def aggregate_logging_outputs(logging_outputs):

        """Aggregate logging outputs from data parallel training."""

        loss_sum = sum(log.get('loss', 0) for log in logging_outputs)

        ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)

        sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)

        return {

            'loss': sum(log.get('loss', 0) for log in logging_outputs) / sample_size / math.log(2),

        agg_output = {

            'loss': loss_sum / sample_size / math.log(2),

        }

        if sample_size != ntokens:

            agg_output['nll_loss'] = loss_sum / ntokens / math.log(2)

        return agg_output

fairseq/criterions/fairseq_criterion.py

+1 −1
Changes for fairseq/criterions/fairseq_criterion.py: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ class FairseqCriterion(_Loss): 
        self.args = args

        self.padding_idx = dst_dict.pad()



    def forward(self, model, sample):

    def forward(self, model, sample, reduce=True):

        """Compute the loss for the given sample.



        Returns a tuple with three elements:

fairseq/criterions/label_smoothed_cross_entropy.py

+16 −6
Changes for fairseq/criterions/label_smoothed_cross_entropy.py: 16 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -11,13 +11,15 @@ import torch 
from torch.autograd.variable import Variable

import torch.nn.functional as F



from fairseq import utils



from .fairseq_criterion import FairseqCriterion





class LabelSmoothedNLLLoss(torch.autograd.Function):



    @staticmethod

    def forward(ctx, input, target, eps, padding_idx, weights):

    def forward(ctx, input, target, eps, padding_idx, weights, reduce=True):

        grad_input = input.new(input.size()).zero_()

        target = target.view(target.size(0), 1)

        grad_input = grad_input.scatter_(grad_input.dim() - 1, target, eps - 1)
@@ -34,11 +36,14 @@ class LabelSmoothedNLLLoss(torch.autograd.Function): 
        grad_input = grad_input.add(-eps / norm)



        ctx.grad_input = grad_input

        if reduce:

            return input.new([grad_input.view(-1).dot(input.view(-1))])

        else:

            return grad_input * input



    @staticmethod

    def backward(ctx, grad):

        return Variable(ctx.grad_input, volatile=True) * grad, None, None, None, None

        return utils.volatile_variable(ctx.grad_input) * grad, None, None, None, None, None





class LabelSmoothedCrossEntropyCriterion(FairseqCriterion):
@@ -48,7 +53,7 @@ class LabelSmoothedCrossEntropyCriterion(FairseqCriterion): 
        self.eps = args.label_smoothing

        self.weights = weights



    def forward(self, model, sample):

    def forward(self, model, sample, reduce=True):

        """Compute the loss for the given sample.



        Returns a tuple with three elements:
@@ -57,12 +62,15 @@ class LabelSmoothedCrossEntropyCriterion(FairseqCriterion): 
        3) logging outputs to display while training

        """

        net_output = model(**sample['net_input'])

        input = F.log_softmax(net_output.view(-1, net_output.size(-1)))

        lprobs = model.get_normalized_probs(net_output, log_probs=True)

        target = sample['target'].view(-1)

        loss = LabelSmoothedNLLLoss.apply(input, target, self.eps, self.padding_idx, self.weights)

        loss = LabelSmoothedNLLLoss.apply(lprobs, target, self.eps, self.padding_idx, self.weights, reduce)

        nll_loss = F.nll_loss(lprobs, target, size_average=False, ignore_index=self.padding_idx, reduce=reduce)

        sample_size = sample['target'].size(0) if self.args.sentence_avg else sample['ntokens']

        logging_output = {

            'loss': loss.data[0],

            'loss': loss.data[0] if reduce else loss.data,

            'nll_loss': nll_loss.data[0] if reduce else loss.data,

            'ntokens': sample['ntokens'],

            'sample_size': sample_size,

        }

        return loss, sample_size, logging_output
@@ -70,7 +78,9 @@ class LabelSmoothedCrossEntropyCriterion(FairseqCriterion): 
    @staticmethod

    def aggregate_logging_outputs(logging_outputs):

        """Aggregate logging outputs from data parallel training."""

        ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)

        sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)

        return {

            'loss': sum(log.get('loss', 0) for log in logging_outputs) / sample_size / math.log(2),

            'nll_loss': sum(log.get('nll_loss', 0) for log in logging_outputs) / ntokens / math.log(2),

        }