Commit 5bf07724 authored by alexeib's avatar alexeib Committed by Myle Ott
Browse files

sort descending when evaluating lm because it is faster (17k wps vs 11k) and will fail early if oom

parent a091d239

fairseq/data/monolingual_dataset.py

+1 −1
Changes for fairseq/data/monolingual_dataset.py: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -72,7 +72,7 @@ class MonolingualDataset(FairseqDataset): 
            order = [np.random.permutation(len(self))]

        else:

            order = [np.arange(len(self))]

        order.append(self.sizes)

        order.append(np.flip(self.sizes, 0))

        return np.lexsort(order)



    def valid_size(self, index, max_positions):