disable final layer norm for transformer decoder as it makes things worse

        else:

            embed_tokens = Embedding(len(task.dictionary), args.decoder_input_dim, task.dictionary.pad())

        decoder = TransformerDecoder(args, task.dictionary, embed_tokens, no_encoder_attn=True)

        decoder = TransformerDecoder(args, task.dictionary, embed_tokens, no_encoder_attn=True, final_norm=False)

        return TransformerLanguageModel(decoder)

class TransformerDecoder(FairseqIncrementalDecoder):

    """Transformer decoder."""

    def __init__(self, args, dictionary, embed_tokens, no_encoder_attn=False, left_pad=False):

    def __init__(self, args, dictionary, embed_tokens, no_encoder_attn=False, left_pad=False, final_norm=True):

        super().__init__(dictionary)

        self.dropout = args.dropout

        self.share_input_output_embed = args.share_decoder_input_output_embed

            self.embed_out = nn.Parameter(torch.Tensor(len(dictionary), output_embed_dim))

            nn.init.normal_(self.embed_out, mean=0, std=output_embed_dim ** -0.5)

        self.register_buffer('version', torch.Tensor([2]))

        self.normalize = args.decoder_normalize_before

        self.normalize = args.decoder_normalize_before and final_norm

        if self.normalize:

           self.layer_norm = LayerNorm(embed_dim)