Commit 42be3ebd authored by Myle Ott's avatar Myle Ott Committed by Facebook Github Bot
Browse files

Merge internal changes (#483) 

Summary:
Changelog:
- `4889802`: can now remove detokenize sentencepiece output with `--remove-bpe=sentencepiece` (fixes #331). Also added `--sacrebleu` for computing detokenized BLEU.
- `0d76427`: fix assertion error when training language model with dataset containing empty sentences
- minor bug and style fixes
Pull Request resolved: https://github.com/pytorch/fairseq/pull/483

Differential Revision: D13867899

Pulled By: myleott

fbshipit-source-id: 25c940b847fe270262ac8f5ac838407b3977fdda
parent 66ce2175

fairseq/bleu.py

+25 −0
Original line number Diff line number Diff line
@@ -35,6 +35,31 @@ class BleuStat(ctypes.Structure): 
    ]





class SacrebleuScorer(object):

    def __init__(self):

        import sacrebleu

        self.sacrebleu = sacrebleu

        self.reset()



    def reset(self, one_init=False):

        if one_init:

            raise NotImplementedError

        self.ref = []

        self.sys = []



    def add_string(self, ref, pred):

        self.ref.append(ref)

        self.sys.append(pred)



    def score(self, order=4):

        return self.result_string(order).bleu



    def result_string(self, order=4):

        if order != 4:

            raise NotImplementedError

        return self.sacrebleu.corpus_bleu(self.sys, [self.ref])





class Scorer(object):

    def __init__(self, pad, eos, unk):

        self.stat = BleuStat()

fairseq/data/data_utils.py

+2 −1
Original line number Diff line number Diff line
@@ -94,7 +94,8 @@ def filter_by_size(indices, size_fn, max_positions, raise_exception=False): 
            return all(

                all(a is None or b is None or a <= b

                    for a, b in zip(idx_size[key], max_positions[key]))

                       for key in intersect_keys)

                for key in intersect_keys

            )

        else:

            return all(a is None or b is None or a <= b

                       for a, b in zip(size_fn(idx), max_positions))

fairseq/data/dictionary.py

+5 −1
Original line number Diff line number Diff line
@@ -57,8 +57,12 @@ class Dictionary(object): 
            else:

                return self[i]



        if bpe_symbol == 'sentencepiece':

            sent = ''.join(token_string(i) for i in tensor if i != self.eos())

        if bpe_symbol is not None:

            sent = sent.replace('\u2581', ' ').strip()

        else:

            sent = ' '.join(token_string(i) for i in tensor if i != self.eos())

        if bpe_symbol is not None and bpe_symbol != 'sentencepiece':

            sent = (sent + ' ').replace(bpe_symbol, '').rstrip()

        return sent

fairseq/data/token_block_dataset.py

+2 −3
Original line number Diff line number Diff line
@@ -66,11 +66,9 @@ class TokenBlockDataset(FairseqDataset): 
            if curr_size > 0:

                self.slice_indices.append((tok_idx, tok_idx + curr_size))

        elif break_mode == 'eos':

            self.slice_indices = np.empty((sum(sizes > 1), 2), dtype=int)

            self.slice_indices = np.empty((len(sizes), 2), dtype=int)

            curr = 0

            for i, sz in enumerate(sizes):

                # skip samples with just 1 example (which would be just the eos token)

                if sz > 1:

                self.slice_indices[i] = (curr, curr + sz)

                curr += sz

        else:
@@ -78,6 +76,7 @@ class TokenBlockDataset(FairseqDataset): 


        self.sizes = np.array([e - s for s, e in self.slice_indices])

        self.slice_indices = np.array(self.slice_indices, dtype=int)



        # build index mapping block indices to the underlying dataset indices

        self.block_to_dataset_index = np.empty((len(self.slice_indices), 3), dtype=int)

        ds_idx, ds_remaining = -1, 0

fairseq/models/fairseq_model.py

+1 −1
Original line number Diff line number Diff line
@@ -29,7 +29,7 @@ class BaseFairseqModel(nn.Module): 
    @classmethod

    def build_model(cls, args, task):

        """Build a new model instance."""

        raise NotImplementedError

        raise NotImplementedError('FairseqModels must implement the build_model method')



    def get_targets(self, sample, net_output):

        """Get targets from either the sample or the net's output."""