Nits (cf1c64a5) · Commits · Simon Will / fairseq

fairseq/data/data_utils.py

+1 −4

Changes for fairseq/data/data_utils.py: 1 added line, 4 removed lines.
Original line number	Diff line number	Diff line
		@@ -81,10 +81,7 @@ def collate_tokens(values, pad_idx, eos_idx, left_pad, move_eos_to_beginning=Fal
		dst.copy_(src)

		for i, v in enumerate(values):
		if left_pad:
		copy_tensor(v, res[i][size - len(v):])
		else:
		copy_tensor(v, res[i][:len(v)])
		copy_tensor(v, res[i][size - len(v):] if left_pad else res[i][:len(v)])
		return res

fairseq/data/indexed_dataset.py

+4 −5

Changes for fairseq/data/indexed_dataset.py: 4 added lines, 5 removed lines.
Original line number	Diff line number	Diff line
		@@ -5,9 +5,10 @@
		# the root directory of this source tree. An additional grant of patent rights
		# can be found in the PATENTS file in the same directory.

		import numpy as np
		import os
		import struct

		import numpy as np
		import torch
		import torch.utils.data

		@@ -197,10 +198,8 @@ class IndexedDatasetBuilder(object):
		index = open(index_file, 'wb')
		index.write(b'TNTIDX\x00\x00')
		index.write(struct.pack('<Q', 1))
		index.write(struct.pack('<QQ', code(self.dtype),
		self.element_size))
		index.write(struct.pack('<QQ', len(self.data_offsets) - 1,
		len(self.sizes)))
		index.write(struct.pack('<QQ', code(self.dtype), self.element_size))
		index.write(struct.pack('<QQ', len(self.data_offsets) - 1, len(self.sizes)))
		write_longs(index, self.dim_offsets)
		write_longs(index, self.data_offsets)
		write_longs(index, self.sizes)

fairseq/models/init.py

+1 −0

Changes for fairseq/models/__init__.py: 1 added line, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -12,6 +12,7 @@ from .fairseq_decoder import FairseqDecoder # noqa: F401
		from .fairseq_encoder import FairseqEncoder # noqa: F401
		from .fairseq_incremental_decoder import FairseqIncrementalDecoder # noqa: F401
		from .fairseq_model import BaseFairseqModel, FairseqModel, FairseqLanguageModel # noqa: F401

		from .composite_encoder import CompositeEncoder # noqa: F401

		MODEL_REGISTRY = {}

fairseq/models/fconv.py

+8 −3

Changes for fairseq/models/fconv.py: 8 added lines, 3 removed lines.
Original line number	Diff line number	Diff line
		@@ -12,10 +12,15 @@ import torch.nn.functional as F

		from fairseq import options, utils
		from fairseq.data.consts import LEFT_PAD_SOURCE, LEFT_PAD_TARGET
		from fairseq.modules import BeamableMM, GradMultiply, LearnedPositionalEmbedding, LinearizedConvolution, AdaptiveSoftmax
		from fairseq.modules import (
		AdaptiveSoftmax, BeamableMM, GradMultiply, LearnedPositionalEmbedding,
		LinearizedConvolution,
		)

		from . import FairseqEncoder, FairseqIncrementalDecoder, FairseqModel, FairseqLanguageModel, register_model, \
		register_model_architecture
		from . import (
		FairseqEncoder, FairseqIncrementalDecoder, FairseqModel,
		FairseqLanguageModel, register_model, register_model_architecture,
		)


		@register_model('fconv')

fairseq/models/fconv_self_att.py

+73 −52

Changes for fairseq/models/fconv_self_att.py: 73 added lines, 52 removed lines.
Original line number	Diff line number	Diff line
		@@ -7,16 +7,23 @@
		#

		import math

		import torch
		import torch.nn as nn
		import torch.nn.functional as F

		from fairseq.data import LanguagePairDataset
		from fairseq.data.consts import LEFT_PAD_SOURCE, LEFT_PAD_TARGET
		from fairseq.modules import GradMultiply, LearnedPositionalEmbedding, LinearizedConvolution, DownsampledMultiHeadAttention
		from fairseq.modules import (
		DownsampledMultiHeadAttention, GradMultiply, LearnedPositionalEmbedding,
		LinearizedConvolution,
		)
		from fairseq import utils

		from . import FairseqEncoder, CompositeEncoder, FairseqDecoder, FairseqModel, register_model, register_model_architecture
		from . import (
		FairseqEncoder, CompositeEncoder, FairseqDecoder, FairseqModel,
		register_model, register_model_architecture,
		)


		@register_model('fconv_self_att')
		class FConvModelSelfAtt(FairseqModel):
		@@ -76,7 +83,6 @@ class FConvModelSelfAtt(FairseqModel):
		pretrained = eval(args.pretrained)
		if pretrained:
		print("\| Loading pretrained model")
		state = torch.load(args.pretrained_checkpoint)
		trained_model = utils.load_ensemble_for_inference(
		# not actually for inference, but loads pretrained model parameters
		filenames=[args.pretrained_checkpoint],
		@@ -131,9 +137,11 @@ class FConvModelSelfAtt(FairseqModel):

		class FConvEncoder(FairseqEncoder):
		"""Convolutional encoder"""
		def __init__(self, dictionary, embed_dim=512, max_positions=1024,
		def __init__(
		self, dictionary, embed_dim=512, max_positions=1024,
		convolutions=((512, 3),) * 20, dropout=0.1, attention=False,
		attention_nheads=1):
		attention_nheads=1,
		):
		super().__init__(dictionary)
		self.dropout = dropout
		self.num_attention_layers = None
		@@ -163,20 +171,20 @@ class FConvEncoder(FairseqEncoder):
		self.attention = nn.ModuleList()
		self.attproj = nn.ModuleList()
		for i, (out_channels, kernel_size) in enumerate(convolutions):
		self.projections.append(Linear(in_channels, out_channels)
		if in_channels != out_channels else None)
		self.projections.append(
		Linear(in_channels, out_channels) if in_channels != out_channels else None
		)
		self.convolutions.append(
		ConvTBC(in_channels, out_channels * 2, kernel_size,
		dropout=dropout))
		ConvTBC(in_channels, out_channels * 2, kernel_size, dropout=dropout)
		)

		self.attention.append(SelfAttention(out_channels, embed_dim,
		attention_nheads)
		if attention[i] else None)
		self.attention.append(
		SelfAttention(out_channels, embed_dim, attention_nheads) if attention[i] else None
		)
		in_channels = out_channels

		self.fc2 = Linear(in_channels, embed_dim)


		def forward(self, src_tokens, src_lengths):
		# embed tokens and positions
		x = self.embed_tokens(src_tokens) + self.embed_positions(src_tokens)
		@@ -226,18 +234,20 @@ class FConvEncoder(FairseqEncoder):

		class FConvDecoder(FairseqDecoder):
		"""Convolutional decoder"""
		def __init__(self, dictionary, embed_dim=512, out_embed_dim=256,
		max_positions=1024, convolutions=((512, 3),) * 8,
		attention=True, dropout=0.1, selfattention=False,
		attention_nheads=1, selfattention_nheads=1,
		def __init__(
		self, dictionary, embed_dim=512, out_embed_dim=256, max_positions=1024,
		convolutions=((512, 3),) * 8, attention=True, dropout=0.1,
		selfattention=False, attention_nheads=1, selfattention_nheads=1,
		project_input=False, gated_attention=False, downsample=False,
		pretrained=False, trained_decoder=None):
		pretrained=False, trained_decoder=None,
		):
		super().__init__(dictionary)
		self.register_buffer('version', torch.Tensor([2]))
		self.pretrained = pretrained
		self.pretrained_decoder = trained_decoder
		self.dropout = dropout
		in_channels = convolutions[0][0]

		def expand_bool_array(val):
		if isinstance(val, bool):
		# expand True into [True, True, ...] and do the same with False
		@@ -269,27 +279,33 @@ class FConvDecoder(FairseqDecoder):
		self.selfattention = nn.ModuleList()
		self.attproj = nn.ModuleList()
		for i, (out_channels, kernel_size) in enumerate(convolutions):
		pad = kernel_size - 1
		self.projections.append(Linear(in_channels, out_channels)
		if in_channels != out_channels else None)
		self.projections.append(
		Linear(in_channels, out_channels) if in_channels != out_channels else None
		)
		self.convolutions.append(
		LinearizedConv1d(in_channels, out_channels * 2, kernel_size,
		padding=(kernel_size - 1), dropout=dropout))

		self.attention.append(DownsampledMultiHeadAttention(out_channels, embed_dim,
		attention_nheads,
		project_input=project_input,
		gated=False, downsample=False)
		if attention[i] else None)

		self.attproj.append(Linear(out_channels, embed_dim, dropout=dropout)
		if attention[i] else None)
		self.selfattention.append(SelfAttention(out_channels, embed_dim,
		selfattention_nheads,
		project_input=project_input,
		gated=gated_attention,
		downsample=downsample)
		if selfattention[i] else None)
		LinearizedConv1d(
		in_channels, out_channels * 2, kernel_size,
		padding=(kernel_size - 1), dropout=dropout,
		)
		)

		self.attention.append(
		DownsampledMultiHeadAttention(
		out_channels, embed_dim, attention_nheads,
		project_input=project_input, gated=False, downsample=False,
		) if attention[i] else None
		)

		self.attproj.append(
		Linear(out_channels, embed_dim, dropout=dropout) if attention[i] else None
		)
		self.selfattention.append(
		SelfAttention(
		out_channels, embed_dim, selfattention_nheads,
		project_input=project_input, gated=gated_attention,
		downsample=downsample,
		) if selfattention[i] else None
		)
		in_channels = out_channels

		self.fc2 = Linear(in_channels, out_embed_dim)
		@@ -301,24 +317,27 @@ class FConvDecoder(FairseqDecoder):
		self.gate1 = nn.Sequential(Linear(out_embed_dim*2, out_embed_dim), nn.Sigmoid())
		self.gate2 = nn.Sequential(Linear(out_embed_dim*2, out_embed_dim), nn.Sigmoid())
		# pretrained and trained models are joined
		self.joining = nn.Sequential(Linear(out_embed_dim2, out_embed_dim2),
		self.joining = nn.Sequential(
		Linear(out_embed_dim2, out_embed_dim2),
		nn.LayerNorm(out_embed_dim*2),
		nn.GLU(),
		Linear(out_embed_dim, out_embed_dim*2),
		nn.LayerNorm(out_embed_dim*2),
		nn.GLU(),
		Linear(out_embed_dim, out_embed_dim),
		nn.LayerNorm(out_embed_dim))
		nn.LayerNorm(out_embed_dim)
		)
		# pretrained model contains an output layer that is nhid -> vocab size
		# but the models are combined in their hidden state
		# the hook stores the output of the pretrained model forward
		self.pretrained_outputs = {}

		def save_output():
		def hook(a, b, output):
		self.pretrained_outputs["out"] = output
		return hook
		self.pretrained_decoder.fc2.register_forward_hook(save_output())

		self.pretrained_decoder.fc2.register_forward_hook(save_output())

		def forward(self, prev_output_tokens, encoder_out_dict):
		encoder_out = encoder_out_dict['encoder']['encoder_out']
		@@ -342,11 +361,9 @@ class FConvDecoder(FairseqDecoder):

		# temporal convolutions
		avg_attn_scores = None
		for proj, conv, attention, selfattention, attproj in zip(self.projections,
		self.convolutions,
		self.attention,
		self.selfattention,
		self.attproj):
		for proj, conv, attention, selfattention, attproj in zip(
		self.projections, self.convolutions, self.attention, self.selfattention, self.attproj
		):
		residual = x if proj is None else proj(x)

		x = F.dropout(x, p=self.dropout, training=self.training)
		@@ -398,11 +415,14 @@ class FConvDecoder(FairseqDecoder):

		def reorder_encoder_out(self, encoder_out_dict, new_order):
		encoder_out_dict['encoder']['encoder_out'] = tuple(
		eo.index_select(0, new_order) for eo in encoder_out_dict['encoder']['encoder_out'])
		eo.index_select(0, new_order) for eo in encoder_out_dict['encoder']['encoder_out']
		)

		if 'pretrained' in encoder_out_dict:
		encoder_out_dict['pretrained']['encoder']['encoder_out'] = tuple(
		eo.index_select(0, new_order) for eo in encoder_out_dict['pretrained']['encoder']['encoder_out'])
		eo.index_select(0, new_order)
		for eo in encoder_out_dict['pretrained']['encoder']['encoder_out']
		)

		return encoder_out_dict

		@@ -425,8 +445,10 @@ class SelfAttention(nn.Module):

		def __init__(self, out_channels, embed_dim, num_heads, project_input=False, gated=False, downsample=False):
		super().__init__()
		self.attention = DownsampledMultiHeadAttention(out_channels, embed_dim, num_heads,
		dropout=0, bias=True, project_input=project_input, gated=gated, downsample=downsample)
		self.attention = DownsampledMultiHeadAttention(
		out_channels, embed_dim, num_heads, dropout=0, bias=True,
		project_input=project_input, gated=gated, downsample=downsample,
		)
		self.in_proj_q = Linear(out_channels, embed_dim)
		self.in_proj_k = Linear(out_channels, embed_dim)
		self.in_proj_v = Linear(out_channels, embed_dim)
		@@ -441,7 +463,6 @@ class SelfAttention(nn.Module):
		return self.ln(x + residual)



		def Embedding(num_embeddings, embedding_dim, padding_idx):
		m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
		m.weight.data.normal_(0, 0.1)