Commit ff68a9ef authored by Myle Ott's avatar Myle Ott
Browse files

Add FairseqTask 

A Task defines the data format, stores shared state (e.g., dictionaries) and provides helpers for building the model/criterion and calculating the loss.

Changes:
- Add TranslationTask and LanguageModelingTask. New tasks can be registered with @register_task decorator.
- Add EpochBatchIterator to encapsulate batching and saving/restoring dataloader position
- Remove LEFT_PAD_* constants and make them configurable per task
parent 2de93532

eval_lm.py

+18 −20
Changes for eval_lm.py: 18 added lines, 20 removed lines.
Original line number Diff line number Diff line
@@ -9,45 +9,43 @@ 
import numpy as np

import torch



from fairseq import options, utils, progress_bar

from fairseq.data import data_utils, data_loaders

from fairseq import data, options, progress_bar, tasks, utils

from fairseq.meters import StopwatchMeter, TimeMeter

from fairseq.sequence_scorer import SequenceScorer





def main(args):

    assert args.path is not None, '--path required for evaluation!'

    print(args)



    if args.max_target_positions is None:

        args.max_target_positions = 1024

    if args.tokens_per_sample is None:

        args.tokens_per_sample = 1024

    print(args)



    use_cuda = torch.cuda.is_available() and not args.cpu

    dataset = data_loaders.load_dataset(args, [args.gen_subset], False)



    # Load dataset splits

    task = tasks.setup_task(args)

    task.load_dataset(args.gen_subset)

    print('| {} {} {} examples'.format(args.data, args.gen_subset, len(task.dataset(args.gen_subset))))



    # Load ensemble

    print('| loading model(s) from {}'.format(args.path))

    models, _ = utils.load_ensemble_for_inference(args.path.split(','), dataset.src_dict, dataset.dst_dict)



    print('| Dictionary: {} types'.format(len(dataset.src_dict)))

    print('| {} {} {} examples'.format(args.data, args.gen_subset, len(dataset.splits[args.gen_subset])))

    models, _ = utils.load_ensemble_for_inference(args.path.split(','), task)



    # Optimize ensemble for generation and set the source and dest dicts on the model (required by scorer)

    for model in models:

        model.make_generation_fast_()

        model.src_dict = dataset.src_dict

        model.dst_dict = dataset.dst_dict



    itr = dataset.eval_dataloader(

        args.gen_subset,

    itr = data.EpochBatchIterator(

        dataset=task.dataset(args.gen_subset),

        max_sentences=args.max_sentences or 4,

        max_positions=args.max_target_positions or 1024,

        descending=True,

    )

    itr = data_utils.ShardedIterator(itr, args.num_shards, args.shard_id)

        max_positions=model.max_positions(),

        num_shards=args.num_shards,

        shard_id=args.shard_id,

    ).next_epoch_itr(shuffle=False)



    gen_timer = StopwatchMeter()

    scorer = SequenceScorer(models)

    scorer = SequenceScorer(models, task.target_dictionary)

    if use_cuda:

        scorer.cuda()
@@ -62,7 +60,7 @@ def main(args): 
                inf_scores = pos_scores.eq(float('inf')) | pos_scores.eq(float('-inf'))

                if inf_scores.any():

                    print('| Skipping tokens with inf scores:',

                          dataset.src_dict.string(hypo['tokens'][inf_scores.nonzero()]))

                          task.target_dictionary.string(hypo['tokens'][inf_scores.nonzero()]))

                    pos_scores = pos_scores[(~inf_scores).nonzero()]

                score_sum += pos_scores.sum()

                count += pos_scores.numel()

examples/language_model/README.md

+5 −5
Changes for examples/language_model/README.md: 5 added lines, 5 removed lines.
Original line number Diff line number Diff line
@@ -22,11 +22,11 @@ $ python preprocess.py --only-source \ 
# Train the model:

# If it runs out of memory, try to reduce max-tokens and max-target-positions

$ mkdir -p checkpoints/wikitext-103

$ python train.py data-bin/wikitext-103 --save-dir /checkpoints/wikitext-103 \ 

  --max-epoch 35 --arch fconv_lm --optimizer nag --lr 1.0 --lr-scheduler reduce_lr_on_plateau --lr-shrink 0.5 \

  --decoder-layers '[(850, 6)] * 3 + [(850,1)] + [(850,5)] * 4 + [(850,1)] + [(850,4)] * 3 + [(1024,4)] + [(2048, 4)]' \ 

  --decoder-embed-dim 280 --clip-norm 0.1 --dropout 0.2 --weight-decay 5e-06 --criterion adaptive_loss \

  --adaptive-softmax-cutoff 10000,20000,200000 --max-tokens 1024 --max-target-positions 1024

$ python train.py --task language_modeling data-bin/wikitext-103 \

  --max-epoch 35 --arch fconv_lm_dauphin_wikitext103 --optimizer nag \

  --lr 1.0 --lr-scheduler reduce_lr_on_plateau --lr-shrink 0.5 \

  --clip-norm 0.1 --dropout 0.2 --weight-decay 5e-06 --criterion adaptive_loss \

  --adaptive-softmax-cutoff 10000,20000,200000 --max-tokens 1024 --tokens-per-sample 1024



# Evaluate:

$ python eval_lm.py data-bin/wikitext-103 --path 'checkpoints/wiki103/checkpoint_best.pt'

examples/stories/README.md

+0 −3
Changes for examples/stories/README.md: 0 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -28,6 +28,3 @@ $ python train.py data-bin/writingPrompts -a fconv_self_att_wp --lr 0.25 --clip- 
# Generate:

$ python generate.py data-bin/writingPrompts --path /path/to/trained/model/checkpoint_best.pt --batch-size 32 --beam 1 --sampling --sampling-topk 10 --sampling-temperature 0.8 --nbest 1 

```

examples/translation/README.md

+0 −1
Changes for examples/translation/README.md: 0 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -103,4 +103,3 @@ $ python generate.py data-bin/fconv_wmt_en_fr \ 
  --path checkpoints/fconv_wmt_en_fr/checkpoint_best.pt --beam 5 --remove-bpe



```

fairseq/criterions/__init__.py

+2 −2
Changes for fairseq/criterions/__init__.py: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -15,8 +15,8 @@ CRITERION_REGISTRY = {} 
CRITERION_CLASS_NAMES = set()





def build_criterion(args, src_dict, dst_dict):

    return CRITERION_REGISTRY[args.criterion](args, src_dict, dst_dict)

def build_criterion(args, task):

    return CRITERION_REGISTRY[args.criterion](args, task)





def register_criterion(name):