fix sinusoidal embedding init size (03fffb98) · Commits · Simon Will / fairseq

fairseq/models/transformer.py

+1 −1

Changes for fairseq/models/transformer.py: 1 added line, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -408,7 +408,7 @@ def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad, le
		nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5)
		nn.init.constant_(m.weight[padding_idx], 0)
		else:
		m = SinusoidalPositionalEmbedding(embedding_dim, padding_idx, left_pad, init_size=num_embeddings)
		m = SinusoidalPositionalEmbedding(embedding_dim, padding_idx, left_pad)
		return m

+3 −3

Changes for fairseq/modules/sinusoidal_positional_embedding.py: 3 added lines, 3 removed lines.
Original line number	Diff line number	Diff line
		@@ -56,12 +56,12 @@ class SinusoidalPositionalEmbedding(nn.Module):
		# recompute/expand embeddings if needed
		bsz, seq_len = input.size()
		max_pos = self.padding_idx + 1 + seq_len
		if max_pos > self.weights.size(0):
		if self.weights is None or max_pos > self.weights.size(0):
		self.weights = SinusoidalPositionalEmbedding.get_embedding(
		max_pos,
		self.embedding_dim,
		self.padding_idx,
		).type_as(self.weights)
		)
		self.weights = self.weights.type_as(self._float_tensor)

		if incremental_state is not None:
		@@ -69,7 +69,7 @@ class SinusoidalPositionalEmbedding(nn.Module):
		return self.weights[self.padding_idx + seq_len, :].expand(bsz, 1, -1)

		positions = utils.make_positions(input.data, self.padding_idx, self.left_pad)
		return self.weights.index_select(0, positions.view(-1)).view(bsz, seq_len, -1)
		return self.weights.index_select(0, positions.view(-1)).view(bsz, seq_len, -1).detach()

		def max_positions(self):
		"""Maximum number of supported positions."""