Left pad source and right pad target (97d7fcb9) · Commits · Simon Will / fairseq

fairseq/data.py

+2 −2

Changes for fairseq/data.py: 2 added lines, 2 removed lines.
Original line number	Diff line number	Diff line
		@@ -142,8 +142,8 @@ def skip_group_enumerator(it, ngpus, offset=0):
		class LanguagePairDataset(object):

		# padding constants
		LEFT_PAD_SOURCE = False
		LEFT_PAD_TARGET = True
		LEFT_PAD_SOURCE = True
		LEFT_PAD_TARGET = False

		def __init__(self, src, dst, pad_idx, eos_idx):
		self.src = src

+2 −6

Changes for fairseq/sequence_generator.py: 2 added lines, 6 removed lines.
Original line number	Diff line number	Diff line
		@@ -61,10 +61,6 @@ class SequenceGenerator(object):
		cuda_device: GPU on which to do generation.
		timer: StopwatchMeter for timing generations.
		"""

		def lstrip_pad(tensor):
		return tensor[tensor.eq(self.pad).sum():]

		if maxlen_b is None:
		maxlen_b = self.maxlen

		@@ -80,8 +76,8 @@ class SequenceGenerator(object):
		timer.stop(s['ntokens'])
		for i, id in enumerate(s['id']):
		src = input['src_tokens'].data[i, :]
		# remove padding from ref, which appears at the beginning
		ref = lstrip_pad(s['target'].data[i, :])
		# remove padding from ref
		ref = utils.rstrip_pad(s['target'].data[i, :], self.pad)
		yield id, src, ref, hypos[i]

		def generate(self, src_tokens, beam_size=None, maxlen=None):

+11 −0

Changes for fairseq/utils.py: 11 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -202,3 +202,14 @@ def post_process_prediction(hypo_tokens, src_str, alignment, align_dict, dst_dic
		# Note that the dictionary can be modified inside the method.
		hypo_tokens = tokenizer.Tokenizer.tokenize(hypo_str, dst_dict, add_if_not_exist=True)
		return hypo_tokens, hypo_str, alignment


		def lstrip_pad(tensor, pad):
		return tensor[tensor.eq(pad).sum():]


		def rstrip_pad(tensor, pad):
		strip = tensor.eq(pad).sum()
		if strip > 0:
		return tensor[:-strip]
		return tensor