Commit 4c2ef2de authored by alexeib's avatar alexeib Committed by Myle Ott
Browse files

Conv lm implementation 

This implements convolutional language model from https://arxiv.org/pdf/1612.08083.pdf

There are 3 modes for constructing batches:

- token block: fill each sample with a specified number of tokens without regard for sentence delimiters - this is what was used for training in the paper
- complete: fill each sample with a specified number of tokens but make sure it contains only complete sentences (i.e. if next sentence goes over token block limit, move it to the next sample) - this was used for evaluation in the paper
- eos: one sentence per sample (skip blank lines)

some results:

GCNN-13 - GBW - 37.46
GCNN-14B - GBW - 33.88
GCNN-8 - Wiki103 - 43.76
GCNN-14 - Wiki103 - 35.66

train:

python train.py /private/home/abaevski/data/wiki103 --save-dir /tmp --fp16 --max-epoch 35 --save-interval 1 --save-interval-updates 1000 --keep-interval-updates 25 --arch fconv_lm --optimizer nag --lr 1.0 --lr-scheduler reduce_lr_on_plateau --lr-shrink 0.5 --decoder-embed-dim 280 --decoder-layers '[(850, 6)] * 3 + [(850,1)] + [(850,5)] * 4 + [(850,1)] + [(850,4)] * 3 + [(1024,4)] + [(2048, 4)]' --clip-norm 0.1 --dropout 0.2 --weight-decay 5e-06 --criterion cross_entropy --max-tokens 1024 --max-target-positions 1024 --seed 1 --log-format json --log-interval 500

eval:

python eval_lm.py ~abaevski/data/wiki103 --path '/checkpoint02/abaevski/2018-04-27/lm_wiki.fp16.mxup300000.fconv.adam.lrs=reduce_lr_on_plateau.emb280.layers(850,6)*3+(850,1)+(850,5)*4+(850,1)+(850,4)*3+(1024,1)+(2048,4).lr0.0005.clp0.1.drp0.3.wd0.0.crt=cross_entropy.mxtk2048.smptk256.seed1.ngpu8/checkpoint_last.pt'
parent 4e1ec2d8

eval_lm.py

0 → 100644
+84 −0
Changes for eval_lm.py: 84 added lines, 0 removed lines.
Original line number Diff line number Diff line 
#!/usr/bin/env python3 -u

# Copyright (c) 2017-present, Facebook, Inc.

# All rights reserved.

#

# This source code is licensed under the license found in the LICENSE file in

# the root directory of this source tree. An additional grant of patent rights

# can be found in the PATENTS file in the same directory.



import numpy as np

import torch



from fairseq import options, utils, progress_bar

from fairseq.data import data_utils, data_loaders

from fairseq.meters import StopwatchMeter, TimeMeter

from fairseq.sequence_scorer import SequenceScorer





def main(args):

    assert args.path is not None, '--path required for evaluation!'

    print(args)



    if args.max_target_positions is None:

        args.max_target_positions = 1024



    use_cuda = torch.cuda.is_available() and not args.cpu

    dataset = data_loaders.load_dataset(args, [args.gen_subset], False)



    # Load ensemble

    print('| loading model(s) from {}'.format(', '.join(args.path)))

    models, _ = utils.load_ensemble_for_inference(args.path, dataset.src_dict, dataset.dst_dict)



    print('| Dictionary: {} types'.format(len(dataset.src_dict)))

    print('| {} {} {} examples'.format(args.data, args.gen_subset, len(dataset.splits[args.gen_subset])))



    # Optimize ensemble for generation and set the source and dest dicts on the model (required by scorer)

    for model in models:

        model.make_generation_fast_()

        model.src_dict = dataset.src_dict

        model.dst_dict = dataset.dst_dict



    itr = dataset.eval_dataloader(

        args.gen_subset,

        max_sentences=args.max_sentences or 4,

        max_positions=args.max_target_positions or 1024,

        descending=True,

    )



    if args.num_shards > 1:

        if args.shard_id < 0 or args.shard_id >= args.num_shards:

            raise ValueError('--shard-id must be between 0 and num_shards')

        itr = data_utils.sharded_iterator(itr, args.num_shards, args.shard_id)



    gen_timer = StopwatchMeter()

    scorer = SequenceScorer(models)

    if use_cuda:

        scorer.cuda()



    score_sum = 0.

    count = 0

    with progress_bar.build_progress_bar(args, itr) as t:

        results = scorer.score_batched_itr(t, cuda=use_cuda, timer=gen_timer)

        wps_meter = TimeMeter()

        for _, src_tokens, __, hypos in results:

            for hypo in hypos:

                pos_scores = hypo['positional_scores']

                inf_scores = pos_scores.eq(float('inf')) | pos_scores.eq(float('-inf'))

                if inf_scores.any():

                    print('| Skipping tokens with inf scores:',

                          dataset.src_dict.string(hypo['tokens'][inf_scores.nonzero()]))

                    pos_scores = pos_scores[(~inf_scores).nonzero()]

                score_sum += pos_scores.sum()

                count += pos_scores.numel()

            wps_meter.update(src_tokens.size(0))

            t.log({'wps': round(wps_meter.avg)})



    avg_nll_loss = -score_sum / count

    print('| Evaluated {} tokens in {:.1f}s ({:.2f} tokens/s)'.format(gen_timer.n, gen_timer.sum, 1. / gen_timer.avg))

    print('| Loss: {:.4f}, Perplexity: {:.2f}'.format(avg_nll_loss, np.exp(avg_nll_loss)))





if __name__ == '__main__':

    parser = options.get_eval_lm_parser()

    args = parser.parse_args()

    main(args)

fairseq/criterions/adaptive_loss.py

0 → 100644
+73 −0
Changes for fairseq/criterions/adaptive_loss.py: 73 added lines, 0 removed lines.
Original line number Diff line number Diff line 
# Copyright (c) 2017-present, Facebook, Inc.

# All rights reserved.

#

# This source code is licensed under the license found in the LICENSE file in

# the root directory of this source tree. An additional grant of patent rights

# can be found in the PATENTS file in the same directory.





import math

import torch.nn.functional as F



from fairseq import utils

from . import FairseqCriterion, register_criterion





@register_criterion('adaptive_loss')

class AdaptiveLoss(FairseqCriterion):

    """This is an implementation of the loss function accompanying the adaptive softmax approximation for

    graphical processing units (GPU), described in the paper "Efficient softmax approximation for GPUs"

    (http://arxiv.org/abs/1609.04309)."""



    def __init__(self, args, src_dict, dst_dict):

        super().__init__(args, src_dict, dst_dict)



    def forward(self, model, sample, reduce=True):

        """Compute the loss for the given sample.



        Returns a tuple with three elements:

        1) the loss, as a Variable

        2) the sample size, which is used as the denominator for the gradient

        3) logging outputs to display while training

        """



        assert hasattr(model.decoder, 'adaptive_softmax') and model.decoder.adaptive_softmax is not None

        adaptive_softmax = model.decoder.adaptive_softmax



        net_output = model(**sample['net_input'])

        target = model.get_targets(sample, net_output).view(-1)



        bsz = target.size(0)



        logits, target = adaptive_softmax(net_output[0], target)

        assert len(target) == len(logits)



        loss = net_output[0].new(1 if reduce else bsz).zero_()



        for i in range(len(target)):

            if target[i] is not None:

                assert (target[i].min() >= 0 and target[i].max() <= logits[i].size(1))

                loss += F.cross_entropy(logits[i], target[i], size_average=False, ignore_index=self.padding_idx,

                                        reduce=reduce)



        sample_size = sample['target'].size(0) if self.args.sentence_avg else sample['ntokens']

        logging_output = {

            'loss': utils.item(loss.data) if reduce else loss.data,

            'ntokens': sample['ntokens'],

            'sample_size': sample_size,

        }

        return loss, sample_size, logging_output



    @staticmethod

    def aggregate_logging_outputs(logging_outputs):

        """Aggregate logging outputs from data parallel training."""

        loss_sum = sum(log.get('loss', 0) for log in logging_outputs)

        ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)

        sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)

        agg_output = {

            'loss': loss_sum / sample_size / math.log(2),

            'sample_size': sample_size,

        }

        if sample_size != ntokens:

            agg_output['nll_loss'] = loss_sum / ntokens / math.log(2)

        return agg_output

fairseq/data/__init__.py

0 → 100644
+13 −0
Changes for fairseq/data/__init__.py: 13 added lines, 0 removed lines.
Original line number Diff line number Diff line 
# Copyright (c) 2017-present, Facebook, Inc.

# All rights reserved.

#

# This source code is licensed under the license found in the LICENSE file in

# the root directory of this source tree. An additional grant of patent rights

# can be found in the PATENTS file in the same directory.



from .dictionary import Dictionary

from .token_block_dataset import TokenBlockDataset

from .language_dataset import LanguageDatasets

from .language_pair_dataset import LanguagePairDataset

from .monolingual_dataset import MonolingualDataset

from .offset_dataset import OffsetDataset

fairseq/data/consts.py

0 → 100644
+10 −0
Changes for fairseq/data/consts.py: 10 added lines, 0 removed lines.
Original line number Diff line number Diff line 
# Copyright (c) 2017-present, Facebook, Inc.

# All rights reserved.

#

# This source code is licensed under the license found in the LICENSE file in

# the root directory of this source tree. An additional grant of patent rights

# can be found in the PATENTS file in the same directory.



# padding constants

LEFT_PAD_SOURCE = True

LEFT_PAD_TARGET = False

fairseq/data/data_loaders.py

0 → 100644
+24 −0
Changes for fairseq/data/data_loaders.py: 24 added lines, 0 removed lines.
Original line number Diff line number Diff line 
# Copyright (c) 2017-present, Facebook, Inc.

# All rights reserved.

#

# This source code is licensed under the license found in the LICENSE file in

# the root directory of this source tree. An additional grant of patent rights

# can be found in the PATENTS file in the same directory.



import os



from fairseq.data import LanguagePairDataset, MonolingualDataset

from fairseq.data.data_utils import infer_language_pair





def load_dataset(args, splits, is_raw):

    """ Detect if we have a multi language dataset, or a single language dataset """

    if args.source_lang is None and args.target_lang is None:

        # find language pair automatically

        args.source_lang, args.target_lang = infer_language_pair(args.data, splits)

    if args.source_lang is None and args.target_lang is None and all(

            os.path.exists(os.path.join(args.data, '{}.bin'.format(split))) for split in splits):

        cls = MonolingualDataset

    else:

        cls = LanguagePairDataset

    return cls.create_dataset(args, splits, is_raw)