Commit a4e34985 authored by Haoran Li's avatar Haoran Li Committed by Facebook Github Bot
Browse files

make dictionary optional 

Reviewed By: jingfeidu

Differential Revision: D13104360

fbshipit-source-id: 9636f5ee2721818f98b33af559fa24292534a72f
parent 161d1e06

fairseq/modules/character_token_embedder.py

+5 −1
Original line number Diff line number Diff line
@@ -51,7 +51,11 @@ class CharacterTokenEmbedder(torch.nn.Module): 


        self.projection = nn.Linear(last_dim, word_embed_dim)



        assert vocab is not None or char_inputs, "vocab must be set if not using char inputs"

        self.vocab = None

        if vocab is not None:

            self.set_vocab(vocab, max_char_len)



        self.reset_parameters()



    def set_vocab(self, vocab, max_char_len):
@@ -78,7 +82,7 @@ class CharacterTokenEmbedder(torch.nn.Module): 


    @property

    def padding_idx(self):

        return self.vocab.pad()

        return Dictionary().pad() if self.vocab is None else self.vocab.pad()



    def reset_parameters(self):

        nn.init.xavier_normal_(self.char_embeddings.weight)

fairseq/tasks/language_modeling.py

+3 −0
Original line number Diff line number Diff line
@@ -87,6 +87,9 @@ class LanguageModelingTask(FairseqTask): 
        Args:

            args (argparse.Namespace): parsed command-line arguments

        """

        dictionary = None

        output_dictionary = None

        if args.data:

            dictionary = Dictionary.load(os.path.join(args.data, 'dict.txt'))

            print('| dictionary: {} types'.format(len(dictionary)))

            output_dictionary = dictionary