Commit 1082ba35 authored by Sergey Edunov's avatar Sergey Edunov Committed by Myle Ott
Browse files

Switch to DistributedDataParallelC10d and bump version 0.5.0 -> 0.6.0 

- no more FP16Trainer, we just have an FP16Optimizer wrapper
- most of the distributed code is moved to a new wrapper class called DistributedFairseqModel, which behaves like DistributedDataParallel and a FairseqModel at the same time
- Trainer now requires an extra dummy_batch argument at initialization, which we do fwd/bwd on when there's an uneven number of batches per worker. We hide the gradients from these dummy batches by multiplying the loss by 0
- Trainer.train_step now takes a list of samples, which will allow cleaner --update-freq
parent 311d2c6c

distributed_train.py

+1 −1
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ def main(args): 
                raise e

            except FileNotFoundError as e:  # Slurm is not installed

                pass

    if args.distributed_init_method is None:

    if args.distributed_init_method is None and args.distributed_port is None:

        raise ValueError('--distributed-init-method or --distributed-port '

                         'must be specified for distributed training')

docs/conf.py

+2 −2
Original line number Diff line number Diff line
@@ -60,9 +60,9 @@ github_doc_root = 'https://github.com/pytorch/fairseq/tree/master/docs/' 
# built documents.

#

# The short X.Y version.

version = '0.5.0'

version = '0.6.0'

# The full version, including alpha/beta/rc tags.

release = '0.5.0'

release = '0.6.0'



# The language for content autogenerated by Sphinx. Refer to documentation

# for a list of supported languages.

docs/data.rst

+2 −0
Original line number Diff line number Diff line
@@ -36,5 +36,7 @@ Iterators 
    :members:

.. autoclass:: fairseq.data.EpochBatchIterator

    :members:

.. autoclass:: fairseq.data.GroupedIterator

    :members:

.. autoclass:: fairseq.data.ShardedIterator

    :members:

fairseq/criterions/adaptive_loss.py

+4 −0
Original line number Diff line number Diff line
@@ -54,6 +54,7 @@ class AdaptiveLoss(FairseqCriterion): 
        logging_output = {

            'loss': utils.item(loss.data) if reduce else loss.data,

            'ntokens': sample['ntokens'],

            'nsentences': sample['target'].size(0),

            'sample_size': sample_size,

        }

        return loss, sample_size, logging_output
@@ -63,9 +64,12 @@ class AdaptiveLoss(FairseqCriterion): 
        """Aggregate logging outputs from data parallel training."""

        loss_sum = sum(log.get('loss', 0) for log in logging_outputs)

        ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)

        nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)

        sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)

        agg_output = {

            'loss': loss_sum / sample_size / math.log(2),

            'ntokens': ntokens,

            'nsentences': nsentences,

            'sample_size': sample_size,

        }

        if sample_size != ntokens:

fairseq/criterions/cross_entropy.py

+4 −0
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ class CrossEntropyCriterion(FairseqCriterion): 
        logging_output = {

            'loss': utils.item(loss.data) if reduce else loss.data,

            'ntokens': sample['ntokens'],

            'nsentences': sample['target'].size(0),

            'sample_size': sample_size,

        }

        return loss, sample_size, logging_output
@@ -46,9 +47,12 @@ class CrossEntropyCriterion(FairseqCriterion): 
        """Aggregate logging outputs from data parallel training."""

        loss_sum = sum(log.get('loss', 0) for log in logging_outputs)

        ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)

        nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)

        sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)

        agg_output = {

            'loss': loss_sum / sample_size / math.log(2),

            'ntokens': ntokens,

            'nsentences': nsentences,

            'sample_size': sample_size,

        }

        if sample_size != ntokens: