More fixes

We provide the following pre-trained fully convolutional sequence-to-sequence models:

* [wmt14.en-fr.fconv-py.tar.bz2](https://s3.amazonaws.com/faiseq-py/models/wmt14.en-fr.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) including vocabularies

* [wmt14.en-de.fconv-py.tar.bz2](https://s3.amazonaws.com/faiseq-py/models/wmt14.en-de.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-German](https://nlp.stanford.edu/projects/nmt) including vocabularies

* [wmt14.en-fr.fconv-py.tar.bz2](https://s3.amazonaws.com/fairseq-py/models/wmt14.en-fr.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) including vocabularies

* [wmt14.en-de.fconv-py.tar.bz2](https://s3.amazonaws.com/fairseq-py/models/wmt14.en-de.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-German](https://nlp.stanford.edu/projects/nmt) including vocabularies

In addition, we provide pre-processed and binarized test sets for the models above:

* [wmt14.en-fr.newstest2014.tar.bz2](https://s3.amazonaws.com/fairseq-py/data/wmt14.en-fr.newstest2014.tar.bz2): newstest2014 test set for WMT14 English-French

Generation with the binarized test sets can be run in batch mode as follows, e.g. for English-French on a GTX-1080ti:

```

$ curl https://s3.amazonaws.com/faiseq-py/models/wmt14.en-fr.fconv-py.tar.bz2 | tar xvjf - -C data-bin

$ curl https://s3.amazonaws.com/fairseq-py/models/wmt14.en-fr.fconv-py.tar.bz2 | tar xvjf - -C data-bin

$ curl https://s3.amazonaws.com/fairseq-py/data/wmt14.en-fr.newstest2014.tar.bz2 | tar xvjf - -C data-bin

$ python generate.py data-bin/wmt14.en-fr.newstest2014  \

  --path data-bin/wmt14.en-fr.fconv-py/model.pt \

  --beam 5 --batch-size 128 --remove-bpe | tee /tmp/gen.out

...

| Translated 3003 sentences (95451 tokens) in 136.3s (700.49 tokens/s)

| Timings: setup 0.1s (0.1%), encoder 1.9s (1.4%), decoder 108.9s (79.9%), search_results 0.0s (0.0%), search_prune 12.5s (9.2%)

TODO: update scores (should be same as score.py)

| BLEU4 = 43.43, 68.2/49.2/37.4/28.8 (BP=0.996, ratio=1.004, sys_len=92087, ref_len=92448)

| Translated 3003 sentences (95451 tokens) in 81.3s (1174.33 tokens/s)

| Generate test with beam=5: BLEU4 = 40.23, 67.5/46.4/33.8/25.0 (BP=0.997, ratio=1.003, syslen=80963, reflen=81194)

# Scoring with score.py:

$ grep ^H /tmp/gen.out | cut -f3- | sed 's/@@ //g' > /tmp/gen.out.sys

$ grep ^T /tmp/gen.out | cut -f2- | sed 's/@@ //g' > /tmp/gen.out.ref

$ grep ^H /tmp/gen.out | cut -f3- > /tmp/gen.out.sys

$ grep ^T /tmp/gen.out | cut -f2- > /tmp/gen.out.ref

$ python score.py --sys /tmp/gen.out.sys --ref /tmp/gen.out.ref

TODO: update scores

BLEU4 = 40.55, 67.6/46.5/34.0/25.3 (BP=1.000, ratio=0.998, sys_len=81369, ref_len=81194)

BLEU4 = 40.23, 67.5/46.4/33.8/25.0 (BP=0.997, ratio=1.003, syslen=80963, reflen=81194)

```

# Join the fairseq community

# can be found in the PATENTS file in the same directory.

#

import logging

import os

import torch

import traceback

from torch.autograd import Variable

from torch.serialization import default_restore_location

        try:

            return torch.save(*args, **kwargs)

        except:

            if i == 3:

                raise

            if i == 2:

                logging.error(traceback.format_exc())

def save_checkpoint(args, epoch, batch_offset, model, optimizer, lr_scheduler, val_loss=None):

            display_hypotheses(None, tokens, line, None, hypos[:min(len(hypos), args.nbest)])

    else:

        non_bpe_dict = {}

        def maybe_remove_bpe_and_reindex(tokens):

            """Helper for removing BPE symbols from a tensor of indices.

            If BPE removal is enabled, the returned tensor is reindexed

            using a new dictionary that is created on-the-fly."""

        def maybe_remove_bpe(tokens):

            """Helper for removing BPE symbols from a hypothesis."""

            if not args.remove_bpe:

                return tokens

            assert (tokens == dataset.dst_dict.pad()).sum() == 0

            return torch.IntTensor([

                non_bpe_dict.setdefault(w, len(non_bpe_dict))

                for w in to_sentence(dataset.dst_dict, tokens, bpe_symbol).split(' ')

            ])

            hypo_minus_bpe = to_sentence(dataset.dst_dict, tokens, bpe_symbol)

            return tokenizer.Tokenizer.tokenize(hypo_minus_bpe, dataset.dst_dict, add_if_not_exist=True)

        # Generate and compute BLEU score

        scorer = bleu.Scorer(

            dataset.dst_dict.pad() if not args.remove_bpe else -1,

            dataset.dst_dict.eos() if not args.remove_bpe else -1,

            dataset.dst_dict.unk())

        scorer = bleu.Scorer(dataset.dst_dict.pad(), dataset.dst_dict.eos(), dataset.dst_dict.unk())

        itr = dataset.dataloader(args.gen_subset, batch_size=args.batch_size, max_positions=args.max_positions)

        num_sentences = 0

        with progress_bar(itr, smoothing=0, leave=False) as t:

            for id, src, ref, hypos in translations:

                ref = ref.int().cpu()

                top_hypo = hypos[0]['tokens'].int().cpu()

                scorer.add(maybe_remove_bpe_and_reindex(ref), maybe_remove_bpe_and_reindex(top_hypo))

                scorer.add(maybe_remove_bpe(ref), maybe_remove_bpe(top_hypo))

                display_hypotheses(id, src, None, ref, hypos[:min(len(hypos), args.nbest)])

                wps_meter.update(src.size(0))

def main():

    parser = argparse.ArgumentParser(description='Command-line script for BLEU scoring.')

    parser.add_argument('-s', '--sys', default='-', help='system output')

    parser.add_argument('-r', '--ref', default='', help='references')

    parser.add_argument('-r', '--ref', required=True, help='references')

    parser.add_argument('-o', '--order', default=4, metavar='N',

                        type=int, help='consider ngrams up to this order')

    parser.add_argument('--ignore-case', action='store_true',