Commit c83efd21 authored by Myle Ott's avatar Myle Ott Committed by GitHub
Browse files

Merge pull request #33 from facebookresearch/oss-merge-internal 

Changes:
Add support for NCCL v2
Add support for additional optimizers
SequenceGenerator returns attention matrix
Misc bugfixes (e.g., fixes #32) and cleanup
parents af86c1ac 104cead1

.gitignore

+3 −0
Changes for .gitignore: 3 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -102,3 +102,6 @@ ENV/ 


# mypy

.mypy_cache/



# Generated files

fairseq/temporal_convolution_tbc

fairseq/__init__.py

+2 −0
Changes for fairseq/__init__.py: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -7,3 +7,5 @@ 
#



from .multiprocessing_pdb import pdb



__all__ = ['pdb']

fairseq/bleu.py

+1 −1
Changes for fairseq/bleu.py: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -97,7 +97,7 @@ class Scorer(object): 
    def result_string(self, order=4):

        assert order <= 4, "BLEU scores for order > 4 aren't supported"

        fmt = 'BLEU{} = {:2.2f}, {:2.1f}'

        for i in range(1, order):

        for _ in range(1, order):

            fmt += '/{:2.1f}'

        fmt += ' (BP={:.3f}, ratio={:.3f}, syslen={}, reflen={})'

        bleup = [p * 100 for p in self.precision()[:order]]

fairseq/clib/libbleu/libbleu.cpp

+2 −2
Changes for fairseq/clib/libbleu/libbleu.cpp: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -104,11 +104,11 @@ void bleu_zero_init(bleu_stat* stat) { 


void bleu_one_init(bleu_stat* stat) {

  bleu_zero_init(stat);

  stat->count1 = 1;

  stat->count1 = 0;

  stat->count2 = 1;

  stat->count3 = 1;

  stat->count4 = 1;

  stat->match1 = 1;

  stat->match1 = 0;

  stat->match2 = 1;

  stat->match3 = 1;

  stat->match4 = 1;

fairseq/criterions/cross_entropy.py

+23 −8
Changes for fairseq/criterions/cross_entropy.py: 23 added lines, 8 removed lines.
Original line number Diff line number Diff line
@@ -18,14 +18,29 @@ class CrossEntropyCriterion(FairseqCriterion): 
        super().__init__()

        self.padding_idx = padding_idx



    def prepare(self, samples):

        self.denom = sum(s['ntokens'] if s else 0 for s in samples)



    def forward(self, net_output, sample):

    def forward(self, model, sample):

        """Compute the loss for the given sample.



        Returns a tuple with three elements:

        1) the loss, as a Variable

        2) the sample size, which is used as the denominator for the gradient

        3) logging outputs to display while training

        """

        net_output = model(**sample['net_input'])

        input = net_output.view(-1, net_output.size(-1))

        target = sample['target'].view(-1)

        loss = F.cross_entropy(input, target, size_average=False, ignore_index=self.padding_idx)

        return loss / self.denom



    def aggregate(self, losses):

        return sum(losses) / math.log(2)
 
        sample_size = sample['ntokens']

        logging_output = {

            'loss': loss.data[0],

            'sample_size': sample_size,

        }

        return loss, sample_size, logging_output



    @staticmethod

    def aggregate_logging_outputs(logging_outputs):

        """Aggregate logging outputs from data parallel training."""

        sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)

        return {

            'loss': sum(log.get('loss', 0) for log in logging_outputs) / sample_size / math.log(2),

        }