Merge internal changes

        if args.fp16:

            model.half()

    assert len(models) > 0

    itr = data.EpochBatchIterator(

        dataset=task.dataset(args.gen_subset),

        max_tokens=args.max_tokens,

        max_sentences=args.max_sentences or 4,

        max_positions=model.max_positions(),

        max_tokens=args.max_tokens or 36000,

        max_sentences=args.max_sentences,

        max_positions=models[0].max_positions(),

        num_shards=args.num_shards,

        shard_id=args.shard_id,

        ignore_invalid_inputs=True,

    score_sum = 0.

    count = 0

    if args.remove_bpe is not None:

        bpe_cont = args.remove_bpe.rstrip()

        bpe_toks = set(i for i in range(len(task.dictionary)) if task.dictionary[i].endswith(bpe_cont))

        bpe_len = len(bpe_cont)

    else:

        bpe_toks = None

        bpe_len = 0

    with progress_bar.build_progress_bar(args, itr) as t:

        results = scorer.score_batched_itr(t, cuda=use_cuda, timer=gen_timer)

        wps_meter = TimeMeter()

        for _, src_tokens, __, hypos in results:

            for hypo in hypos:

                pos_scores = hypo['positional_scores']

                skipped_toks = 0

                if bpe_toks is not None:

                    for i in range(len(hypo['tokens']) - 1):

                        if hypo['tokens'][i].item() in bpe_toks:

                            skipped_toks += 1

                            pos_scores[i + 1] += pos_scores[i]

                            pos_scores[i] = 0

                inf_scores = pos_scores.eq(float('inf')) | pos_scores.eq(float('-inf'))

                if inf_scores.any():

                    print('| Skipping tokens with inf scores:',

                          task.target_dictionary.string(hypo['tokens'][inf_scores.nonzero()]))

                    pos_scores = pos_scores[(~inf_scores).nonzero()]

                score_sum += pos_scores.sum()

                count += pos_scores.numel()

                count += pos_scores.numel() - skipped_toks

                if args.output_word_probs:

                    w = ''

                    word_prob = []

                    for i in range(len(hypo['tokens'])):

                        w_ind = hypo['tokens'][i].item()

                        w += task.dictionary[w_ind]

                        if bpe_toks is not None and w_ind in bpe_toks:

                            w = w[:-bpe_len]

                        else:

                            word_prob.append((w, pos_scores[i].item()))

                            w = ''

                    print('\t'.join('{} [{:2f}]'.format(x[0], x[1]) for x in word_prob))

            wps_meter.update(src_tokens.size(0))

            t.log({'wps': round(wps_meter.avg)})

```

To train transformer model on IWSLT'14 German to English:

```

# Preparation steps are the same as for fconv model.

# Train the model (better for a single GPU setup):

$ mkdir -p checkpoints/transformer

$ CUDA_VISIBLE_DEVICES=0 python train.py data-bin/iwslt14.tokenized.de-en \

  -a transformer_iwslt_de_en --optimizer adam --lr 0.0005 -s de -t en \

  --label-smoothing 0.1 --dropout 0.3 --max-tokens 4000 \

  --min-lr '1e-09' --lr-scheduler inverse_sqrt --weight-decay 0.0001 \

  --criterion label_smoothed_cross_entropy --max-update 50000 \

  --warmup-updates 4000 --warmup-init-lr '1e-07' \

  --adam-betas '(0.9, 0.98)' --save-dir checkpoints/transformer

# Average 10 latest checkpoints:

$ python scripts/average_checkpoints.py --inputs checkpoints/transformer \

   --num-epoch-checkpoints 10 --output checkpoints/transformer/model.pt

# Generate:

$ python generate.py data-bin/iwslt14.tokenized.de-en \

  --path checkpoints/transformer/model.pt \

  --batch-size 128 --beam 5 --remove-bpe

```

### prepare-wmt14en2de.sh

from .dictionary import Dictionary

from .fairseq_dataset import FairseqDataset

from .indexed_dataset import IndexedInMemoryDataset, IndexedRawTextDataset

from .indexed_dataset import IndexedDataset, IndexedInMemoryDataset, IndexedRawTextDataset  # noqa: F401

from .language_pair_dataset import LanguagePairDataset

from .monolingual_dataset import MonolingualDataset

from .token_block_dataset import TokenBlockDataset

            self.slice_indices = [block_at(i) for i in range(length)]

        elif break_mode == 'complete':

            assert sizes is not None and sum(sizes) == len(tokens)

            assert sizes is not None and sum(sizes) == len(tokens), '{} != {}'.format(sum(sizes), len(tokens))

            tok_idx = 0

            sz_idx = 0

            curr_size = 0

            if curr_size > 0:

                self.slice_indices.append((tok_idx, tok_idx + curr_size))

        elif break_mode == 'eos':

            assert sizes is not None and sum(sizes) == len(tokens)

            assert sizes is not None and sum(sizes) == len(tokens), '{} != {}'.format(sum(sizes), len(tokens))

            curr = 0

            for sz in sizes:

                # skip samples with just 1 example (which would be just the eos token)

    def __getitem__(self, index):

        s, e = self.slice_indices[index]

        item = torch.LongTensor(self.tokens[s:e])

        if self.include_targets:

            if e == self.total_size:

                return item[:-1], item[1:]

            else:

                return item, torch.LongTensor(self.tokens[s + 1:e + 1])

            # target is the sentence, for source, rotate item one token to the left (would start with eos)

            if s == 0:

                source = np.concatenate([self.tokens[-1:], self.tokens[0:e - 1]])

            else:

                source = self.tokens[s - 1:e - 1]

            return torch.LongTensor(source), item

        return item

    def __len__(self):

    def forward(self, prev_output_tokens, encoder_out):

        raise NotImplementedError

    def get_normalized_probs(self, net_output, log_probs, _):

    def get_normalized_probs(self, net_output, log_probs, sample):

        """Get normalized probabilities (or log probs) from a net's output."""

        if hasattr(self, 'adaptive_softmax') and self.adaptive_softmax is not None:

            assert sample is not None and 'target' in sample

            out = self.adaptive_softmax.get_log_prob(net_output[0], sample['target'])

            return out.exp_() if not log_probs else out

        logits = net_output[0].float()

        if log_probs:

            return F.log_softmax(logits, dim=-1)