Commit 6e4b7e22 authored by Myle Ott's avatar Myle Ott
Browse files

Refactor model definitions 

* Move some functionality out of FConvModel into FairseqModel base class
* Move incremental decoding functionality into FairseqIncrementalDecoder module
* Refactor positional embeddings to be more specific to FConvModel
parent 820f796f

fairseq/criterions/__init__.py

+0 −1
Changes for fairseq/criterions/__init__.py: 0 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -7,7 +7,6 @@ 
#



from .cross_entropy import CrossEntropyCriterion

from .fairseq_criterion import FairseqCriterion

from .label_smoothed_cross_entropy import LabelSmoothedCrossEntropyCriterion



__all__ = [

fairseq/data.py

+16 −30
Changes for fairseq/data.py: 16 added lines, 30 removed lines.
Original line number Diff line number Diff line
@@ -106,11 +106,13 @@ class LanguageDatasets(object): 
                    max_positions=max_positions,

                    sort_by_source_size=sort_by_source_size)

        elif split.startswith('valid'):

            batch_sampler = list(batches_by_size(dataset.src, batch_size, max_tokens, dst=dataset.dst,

            batch_sampler = list(batches_by_size(

                dataset.src, batch_size, max_tokens, dst=dataset.dst,

                max_positions=max_positions,

                ignore_invalid_inputs=skip_invalid_size_inputs_valid_test))

        else:

            batch_sampler = list(batches_by_size(dataset.src, batch_size, max_tokens, max_positions=max_positions,

            batch_sampler = list(batches_by_size(

                dataset.src, batch_size, max_tokens, max_positions=max_positions,

                ignore_invalid_inputs=skip_invalid_size_inputs_valid_test))



        return torch.utils.data.DataLoader(
@@ -137,6 +139,11 @@ def skip_group_enumerator(it, ngpus, offset=0): 




class LanguagePairDataset(object):



    # padding constants

    LEFT_PAD_SOURCE = False

    LEFT_PAD_TARGET = True



    def __init__(self, src, dst, pad_idx, eos_idx):

        self.src = src

        self.dst = dst
@@ -166,17 +173,13 @@ class LanguagePairDataset(object): 
            return LanguagePairDataset.collate_tokens(

                [s[key] for s in samples], pad_idx, eos_idx, left_pad, move_eos_to_beginning)



        def merge_positions(key, left_pad):

            return LanguagePairDataset.collate_positions([s[key] for s in samples], pad_idx, left_pad)



        ntokens = sum(len(s['target']) for s in samples)

        return {

            'id': torch.LongTensor([s['id'].item() for s in samples]),

            'input_tokens': merge('target', left_pad=True, move_eos_to_beginning=True),

            'input_positions': merge_positions('target', left_pad=True),

            'target': merge('target', left_pad=True),

            'src_tokens': merge('source', left_pad=False),

            'src_positions': merge_positions('source', left_pad=False),

            'input_tokens': merge('target', left_pad=LanguagePairDataset.LEFT_PAD_TARGET,

                                  move_eos_to_beginning=True),

            'target': merge('target', left_pad=LanguagePairDataset.LEFT_PAD_TARGET),

            'src_tokens': merge('source', left_pad=LanguagePairDataset.LEFT_PAD_SOURCE),

            'ntokens': ntokens,

        }
@@ -201,18 +204,6 @@ class LanguagePairDataset(object): 
                copy_tensor(v, res[i][:len(v)])

        return res



    @staticmethod

    def collate_positions(values, pad_idx, left_pad):

        start = pad_idx + 1

        size = max(v.size(0) for v in values)

        res = values[0].new(len(values), size).fill_(pad_idx)

        for i, v in enumerate(values):

            if left_pad:

                torch.arange(start, start + len(v), out=res[i][size-len(v):])

            else:

                torch.arange(start, start + len(v), out=res[i][:len(v)])

        return res





def batches_by_size(src, batch_size=None, max_tokens=None, dst=None,

                    max_positions=1024, ignore_invalid_inputs=False):
@@ -243,15 +234,12 @@ def batches_by_size(src, batch_size=None, max_tokens=None, dst=None, 
    cur_max_size = 0

    ignored = []

    for idx in indices:

        # - 2 here stems from make_positions() where we offset positions

        # by padding_value + 1

        if src.sizes[idx] < 2 or \

                (False if dst is None else dst.sizes[idx] < 2) or \

                sizes[idx] > max_positions - 2:

                sizes[idx] > max_positions:

            if ignore_invalid_inputs:

                ignored.append(idx)

                continue



            raise Exception("Unable to handle input id {} of "

                            "size {} / {}.".format(idx, src.sizes[idx],

                                                   "none" if dst is None else dst.sizes[idx]))
@@ -290,11 +278,9 @@ def shuffled_batches_by_size(src, dst, max_tokens=None, epoch=1, sample=0, 
        sample_len = 0

        ignored = []

        for idx in indices:

            # - 2 here stems from make_positions() where we offset positions

            # by padding_value + 1

            if src.sizes[idx] < 2 or dst.sizes[idx] < 2 or \

                            src.sizes[idx] > max_positions - 2 or \

                            dst.sizes[idx] > max_positions - 2:

                    src.sizes[idx] > max_positions or \

                    dst.sizes[idx] > max_positions:

                ignored.append(idx)

                continue

            sample_len = max(sample_len, src.sizes[idx], dst.sizes[idx])

fairseq/dictionary.py

+1 −0
Changes for fairseq/dictionary.py: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ class Dictionary(object): 
        self.symbols = []

        self.count = []

        self.indices = {}

        # dictionary indexing starts at 1 for consistency with Lua

        self.add_symbol('<Lua heritage>')

        self.pad_index = self.add_symbol(pad)

        self.eos_index = self.add_symbol(eos)

fairseq/models/__init__.py

+5 −0
Changes for fairseq/models/__init__.py: 5 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -6,6 +6,11 @@ 
# can be found in the PATENTS file in the same directory.

#



from .fairseq_decoder import FairseqDecoder

from .fairseq_encoder import FairseqEncoder

from .fairseq_incremental_decoder import FairseqIncrementalDecoder

from .fairseq_model import FairseqModel



from . import fconv

fairseq/models/fairseq_decoder.py

0 → 100644
+20 −0
Changes for fairseq/models/fairseq_decoder.py: 20 added lines, 0 removed lines.
Original line number Diff line number Diff line 
# Copyright (c) 2017-present, Facebook, Inc.

# All rights reserved.

#

# This source code is licensed under the license found in the LICENSE file in

# the root directory of this source tree. An additional grant of patent rights

# can be found in the PATENTS file in the same directory.

#



import torch.nn as nn





class FairseqDecoder(nn.Module):

    """Base class for decoders."""



    def __init__(self):

        super().__init__()



    def max_positions(self):

        """Maximum input length supported by the decoder."""

        raise NotImplementedError