Commit a04c4cf4 authored by Myle Ott's avatar Myle Ott
Browse files

Fix --remove-bpe to strip trailing BPE symbols

parent c53b2ee0

fairseq/dictionary.py

+1 −1
Changes for fairseq/dictionary.py: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -59,7 +59,7 @@ class Dictionary(object): 


        sent = ' '.join(token_string(i) for i in tensor if i != self.eos())

        if bpe_symbol is not None:

            sent = sent.replace(bpe_symbol, '')

            sent = (sent + ' ').replace(bpe_symbol, '').rstrip()

        return sent



    def unk_string(self, escape=False):