Commit 678d0bbc authored by alexeib's avatar alexeib Committed by Myle Ott
Browse files

add sinusoidal pos initialization

parent 03fffb98

fairseq/models/transformer.py

+1 −1
Changes for fairseq/models/transformer.py: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -408,7 +408,7 @@ def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad, le 
        nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5)

        nn.init.constant_(m.weight[padding_idx], 0)

    else:

        m = SinusoidalPositionalEmbedding(embedding_dim, padding_idx, left_pad)

        m = SinusoidalPositionalEmbedding(embedding_dim, padding_idx, left_pad, num_embeddings)

    return m