Merge internal changes (#295)

## Replicating results from "Scaling Neural Machine Translation"

To replicate results from the paper [Scaling Neural Machine Translation (Ott et al., 2018)](https://arxiv.org/abs/1806.00187):

To replicate results from the paper [Scaling Neural Machine Translation (Ott et al., 2018)](https://arxiv.org/abs/1806.00187),

please first download the [preprocessed WMT'16 En-De data provided by Google](https://drive.google.com/uc?export=download&id=0B_bZck-ksdkpM25jRUN2X2UxMm8).

1. Prepare the WMT'14 En-De data with a BPE vocab of 32k:

1. Extract the WMT'16 En-De data:

```

$ bash prepare-wmt14en2de.sh --scaling18

$ cd ../..

$ TEXT=wmt16_en_de_bpe32k

$ mkdir $TEXT

$ tar -xzvf wmt16_en_de.tar.gz -C $TEXT

```

2. Preprocess the dataset with a joined dictionary:

```

$ TEXT=examples/translation/wmt14_en_de

$ python preprocess.py --source-lang en --target-lang de \

  --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \

  --destdir data-bin/wmt14_en_de_joined_dict \

  --trainpref $TEXT/train.tok.clean.bpe.32000 \

  --validpref $TEXT/newstest2013.tok.bpe.32000 \

  --testpref $TEXT/newstest2014.tok.bpe.32000 \

  --destdir data-bin/wmt16_en_de_bpe32k \

  --nwordssrc 32768 --nwordstgt 32768 \

  --joined-dictionary

```

3. Train a model:

```

$ python train.py data-bin/wmt14_en_de_joined_dict \

$ python train.py data-bin/wmt16_en_de_bpe32k \

  --arch transformer_vaswani_wmt_en_de_big --share-all-embeddings \

  --optimizer adam --adam-betas '(0.9, 0.98)' --clip-norm 0.0 \

  --lr-scheduler inverse_sqrt --warmup-init-lr 1e-07 --warmup-updates 4000 \

    CORPORA[2]="training/news-commentary-v9.de-en"

fi

# This will make the dataset comparable to the one used in "Scaling Neural Machine Translation"

# https://arxiv.org/abs/1806.00187

if [ "$1" == "--scaling18" ]; then

    BPE_TOKENS=32764

fi

if [ ! -d "$SCRIPTS" ]; then

    echo "Please set SCRIPTS variable correctly to point to Moses scripts."

    exit

    echo ""

done

if [ "$1" == "--scaling18" ]; then

    # apply length filtering before BPE for --scaling18

    perl $CLEAN $tmp/train.tags.$lang.tok $src $tgt $tmp/train 1 80

    # use newstest2013 for valid

    echo "pre-processing valid data..."

    for l in $src $tgt; do

        rm $tmp/valid.$l

        cat $orig/$dev.$l | \

            perl $NORM_PUNC $l | \

            perl $REM_NON_PRINT_CHAR | \

            perl $TOKENIZER -threads 8 -a -l $l >> $tmp/valid.$l

    done

else

echo "splitting train and valid..."

for l in $src $tgt; do

    awk '{if (NR%100 == 0)  print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/valid.$l

    awk '{if (NR%100 != 0)  print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/train.$l

done

fi

TRAIN=$tmp/train.de-en

BPE_CODE=$prep/code

    done

done

if [ "$1" == "--scaling18" ]; then

    for L in $src $tgt; do

        cp $tmp/bpe.train.$L $prep/train.$L

        cp $tmp/bpe.valid.$L $prep/valid.$L

    done

else

perl $CLEAN -ratio 1.5 $tmp/bpe.train $src $tgt $prep/train 1 250

perl $CLEAN -ratio 1.5 $tmp/bpe.valid $src $tgt $prep/valid 1 250

fi

for L in $src $tgt; do

    cp $tmp/bpe.test.$L $prep/test.$L

from .dictionary import Dictionary, TruncatedDictionary

from .fairseq_dataset import FairseqDataset

from .indexed_dataset import IndexedDataset, IndexedInMemoryDataset, IndexedRawTextDataset

from .concat_dataset import ConcatDataset

from .indexed_dataset import IndexedDataset, IndexedCachedDataset, IndexedInMemoryDataset, IndexedRawTextDataset

from .language_pair_dataset import LanguagePairDataset

from .monolingual_dataset import MonolingualDataset

from .token_block_dataset import TokenBlockDataset

)

__all__ = [

    'ConcatDataset',

    'CountingIterator',

    'Dictionary',

    'EpochBatchIterator',

    'FairseqDataset',

    'GroupedIterator',

    'IndexedCachedDataset',

    'IndexedDataset',

    'IndexedInMemoryDataset',

    'IndexedRawTextDataset',

import bisect

from . import FairseqDataset

class ConcatDataset(FairseqDataset):

    @staticmethod

    def cumsum(sequence):

        r, s = [], 0

        for e in sequence:

            l = len(e)

            r.append(l + s)

            s += l

        return r

    def __init__(self, datasets):

        super(ConcatDataset, self).__init__()

        assert len(datasets) > 0, 'datasets should not be an empty iterable'

        self.datasets = list(datasets)

        self.cummulative_sizes = self.cumsum(self.datasets)

    def __len__(self):

        return self.cummulative_sizes[-1]

    def __getitem__(self, idx):

        dataset_idx = bisect.bisect_right(self.cummulative_sizes, idx)

        if dataset_idx == 0:

            sample_idx = idx

        else:

            sample_idx = idx - self.cummulative_sizes[dataset_idx - 1]

        return self.datasets[dataset_idx][sample_idx]

    @property

    def supports_prefetch(self):

        return all([d.supports_prefetch for d in self.datasets])

    def prefetch(self, indices):

        frm = 0

        for to, ds in zip(self.cummulative_sizes, self.datasets):

            ds.prefetch([i - frm for i in indices if frm <= i < to])

            frm = to

    sample_len = 0

    sample_lens = []

    ignored = []

    for idx in indices:

        sample_lens.append(num_tokens_fn(idx))

        sample_len = max(sample_len, sample_lens[-1])