Commit c6d4386c authored Apr 23, 2018 by Alexei Baevski Committed by Myle Ott Jun 15, 2018

Fix embedding initialization for padding

parent 1ec5f0a0

fairseq/models/fconv.py

+4 −2

Changes for fairseq/models/fconv.py: 4 added lines, 2 removed lines.
Original line number	Diff line number	Diff line
		@@ -416,13 +416,15 @@ class FConvDecoder(FairseqIncrementalDecoder):

		def Embedding(num_embeddings, embedding_dim, padding_idx):
		m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
		m.weight.data.normal_(0, 0.1)
		nn.init.normal(m.weight, 0, 0.1)
		nn.init.constant(m.weight[padding_idx], 0)
		return m


		def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad):
		m = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad)
		m.weight.data.normal_(0, 0.1)
		nn.init.normal(m.weight, 0, 0.1)
		nn.init.constant(m.weight[padding_idx], 0)
		return m

fairseq/models/lstm.py

+2 −1

Changes for fairseq/models/lstm.py: 2 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -403,7 +403,8 @@ class LSTMDecoder(FairseqIncrementalDecoder):

		def Embedding(num_embeddings, embedding_dim, padding_idx):
		m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
		m.weight.data.uniform_(-0.1, 0.1)
		nn.init.uniform(m.weight, -0.1, 0.1)
		nn.init.constant(m.weight[padding_idx], 0)
		return m

fairseq/models/transformer.py

+1 −0

Changes for fairseq/models/transformer.py: 1 added line, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -379,6 +379,7 @@ def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad, le
		if learned:
		m = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad)
		nn.init.normal(m.weight, mean=0, std=embedding_dim**-0.5)
		nn.init.constant(m.weight[padding_idx], 0)
		else:
		m = SinusoidalPositionalEmbedding(embedding_dim, padding_idx, left_pad, init_size=num_embeddings)
		return m