Merge internal changes

# This source code is licensed under the license found in the LICENSE file in

# the root directory of this source tree. An additional grant of patent rights

# can be found in the PATENTS file in the same directory.

"""

Evaluate the perplexity of a trained language model.

"""

import numpy as np

import torch

from fairseq import data, options, progress_bar, tasks, utils

from fairseq import options, progress_bar, tasks, utils

from fairseq.meters import StopwatchMeter, TimeMeter

from fairseq.sequence_scorer import SequenceScorer

        self.word = word

        self.is_bpe = is_bpe

        self.log_prob = 0

        self.next_word_prob = 0

        self.count = 0

    def add(self, log_prob):

        self.missing_next_words = 0

    def add(self, log_prob, next_word_prob):

        """ increments counters for the sum of log probs of current word and next

            word (given context ending at current word). Since the next word might be at the end of the example,

            or it might be not counted because it is not an ending subword unit,

            also keeps track of how many of those we have seen """

        if next_word_prob is not None:

            self.next_word_prob += next_word_prob

        else:

            self.missing_next_words += 1

        self.log_prob += log_prob

        self.count += 1

    def __str__(self):

        return '{}\t{}\t{}\t{}'.format(self.word, self.count, self.log_prob / self.count, self.is_bpe)

        return '{}\t{}\t{}\t{}\t{}\t{}'.format(self.word, self.count, self.log_prob, self.is_bpe,

                                               self.next_word_prob, self.count - self.missing_next_words)

def main(parsed_args):

    assert len(models) > 0

    print('num. model params: {}'.format(sum(p.numel() for p in models[0].parameters())))

    itr = task.get_batch_iterator(

        dataset=task.dataset(args.gen_subset),

        max_tokens=args.max_tokens or 36000,

                    print('| Skipping tokens with inf scores:',

                          task.target_dictionary.string(hypo['tokens'][inf_scores.nonzero()]))

                    pos_scores = pos_scores[(~inf_scores).nonzero()]

                score_sum += utils.item(pos_scores.sum())

                score_sum += pos_scores.sum().cpu()

                count += pos_scores.numel() - skipped_toks

                if args.output_word_probs or args.output_word_stats:

                            is_bpe = True

                        else:

                            word_prob.append((w, pos_scores[i].item()))

                            word_stats.setdefault(w, WordStat(w, is_bpe)).add(pos_scores[i].item())

                            next_prob = None

                            ind = i + 1

                            while ind < len(hypo['tokens']):

                                if pos_scores[ind].item() != 0:

                                    next_prob = pos_scores[ind]

                                    break

                                ind += 1

                            word_stats.setdefault(w, WordStat(w, is_bpe)).add(pos_scores[i].item(), next_prob)

                            is_bpe = False

                            w = ''

                    if args.output_word_probs:

from .multiprocessing_pdb import pdb

__all__ = ['pdb']

__version__ = '0.6.0'

import fairseq.criterions

import fairseq.models

# Copyright (c) 2017-present, Facebook, Inc.

# All rights reserved.

#

# This source code is licensed under the license found in the LICENSE file in

# the root directory of this source tree. An additional grant of patent rights

# can be found in the PATENTS file in the same directory.

from torch import nn

from fairseq import utils

from . import FairseqCriterion, register_criterion

@register_criterion('composite_loss')

class CompositeLoss(FairseqCriterion):

    """This is a composite loss that, given a list of model outputs and a list of targets,

    computes an average of losses for each output-target pair"""

    @staticmethod

    def add_args(parser):

        """Add criterion-specific arguments to the parser."""

        parser.add_argument('--underlying-criterion', type=str, metavar='VAL', required=True,

                            help='underlying criterion to use for the composite loss')

    def __init__(self, args, task):

        super().__init__(args, task)

        saved_criterion = args.criterion

        args.criterion = args.underlying_criterion

        assert saved_criterion != args.underlying_criterion

        self.underlying_criterion = task.build_criterion(args)

        args.criterion = saved_criterion

    class FakeModel(nn.Module):

        def __init__(self, model, net_out, target):

            super(CompositeLoss.FakeModel, self).__init__()

            self.model = model

            self.net_out = net_out

            self.target = target

        def forward(self, **unused):

            return self.net_out

        def get_targets(self, *unused):

            return self.target

        @property

        def decoder(self):

            return self.model.decoder

    def forward(self, model, sample, reduce=True):

        net_outputs = model(**sample['net_input'])

        targets = sample['target']

        bsz = targets[0].size(0)

        loss = net_outputs[0][0].new(1 if reduce else bsz).zero_()

        sample_size = 0

        logging_output = {}

        for o, t in zip(net_outputs[0], targets):

            m = CompositeLoss.FakeModel(model, (o, net_outputs[1]), t)

            l, ss, logging_output = self.underlying_criterion(m, sample, reduce)

            loss += l

            sample_size += ss

        loss.div_(len(targets))

        sample_size /= len(targets)

        logging_output['loss'] = utils.item(loss.data) if reduce else loss.data

        return loss, sample_size, logging_output

    def _aggregate_logging_outputs(self, logging_outputs):

        return self.underlying_criterion._aggregate_logging_outputs(logging_outputs)

        """Aggregate logging outputs from data parallel training."""

        raise NotImplementedError

    def _aggregate_logging_outputs(self, logging_outputs):

        """An instance method version of :func:`aggregate_logging_outputs`.

        This can be overridden if needed, but please be careful not to rely

        on shared state when aggregating logging outputs otherwise you may

        get incorrect results.

        """

        return self.__class__.aggregate_logging_outputs(logging_outputs)

    @staticmethod

    def grad_denom(sample_sizes):

        """Compute the gradient denominator for a set of sample sizes."""

import bisect

import numpy as np

from . import FairseqDataset

class ConcatDataset(FairseqDataset):

    @staticmethod

    def cumsum(sequence):

    def cumsum(sequence, sample_ratios):

        r, s = [], 0

        for e in sequence:

            l = len(e)

        for e, ratio in zip(sequence, sample_ratios):

            l = ratio * len(e)

            r.append(l + s)

            s += l

        return r

    def __init__(self, datasets):

    def __init__(self, datasets, sample_ratios=1):

        super(ConcatDataset, self).__init__()

        assert len(datasets) > 0, 'datasets should not be an empty iterable'

        self.datasets = list(datasets)

        self.cummulative_sizes = self.cumsum(self.datasets)

        if isinstance(sample_ratios, int):

            sample_ratios = [sample_ratios] * len(self.datasets)

        self.sample_ratios = sample_ratios

        self.cummulative_sizes = self.cumsum(self.datasets, sample_ratios)

        self.real_sizes = [len(d) for d in self.datasets]

    def __len__(self):

        return self.cummulative_sizes[-1]

            sample_idx = idx

        else:

            sample_idx = idx - self.cummulative_sizes[dataset_idx - 1]

        sample_idx = sample_idx % self.real_sizes[dataset_idx]

        return self.datasets[dataset_idx][sample_idx]

    @property

    def sizes(self):

        return np.concatenate([np.tile(ds.sizes, sr) for ds, sr in zip(self.datasets, self.sample_ratios)])

    @property

    def supports_prefetch(self):

        return all([d.supports_prefetch for d in self.datasets])

    def prefetch(self, indices):

        frm = 0

        for to, ds in zip(self.cummulative_sizes, self.datasets):

            ds.prefetch([i - frm for i in indices if frm <= i < to])

            real_size = len(ds)

            ds.prefetch([(i - frm) % real_size for i in indices if frm <= i < to])

            frm = to