Commit 45082e48 authored by alexeib's avatar alexeib Committed by Myle Ott
Browse files

make batching faster for monolingual dataset

parent 0b5166db

fairseq/data/monolingual_dataset.py

+1 −2
Changes for fairseq/data/monolingual_dataset.py: 1 added line, 2 removed lines.
Original line number Diff line number Diff line
@@ -63,8 +63,7 @@ class MonolingualDataset(FairseqDataset): 


    def num_tokens(self, index):

        """Return an example's length (number of tokens), used for batching."""

        source, target = self.dataset[index]

        return len(source)

        return self.sizes[index]



    def ordered_indices(self):

        """Ordered indices for batching."""