Commit 66262a38 authored by Myle Ott's avatar Myle Ott Committed by Facebook Github Bot
Browse files

Use --workers for validation sets in preprocess.py 

Summary: Pull Request resolved: https://github.com/pytorch/fairseq/pull/550

Differential Revision: D14286008

Pulled By: myleott

fbshipit-source-id: 6055acf98023fdd01f85ac3d7c4e7fb786e54389
parent 92a6c548

preprocess.py

+2 −2
Original line number Diff line number Diff line
@@ -178,11 +178,11 @@ def main(args): 
        if args.validpref:

            for k, validpref in enumerate(args.validpref.split(",")):

                outprefix = "valid{}".format(k) if k > 0 else "valid"

                make_dataset(vocab, validpref, outprefix, lang)

                make_dataset(vocab, validpref, outprefix, lang, num_workers=args.workers)

        if args.testpref:

            for k, testpref in enumerate(args.testpref.split(",")):

                outprefix = "test{}".format(k) if k > 0 else "test"

                make_dataset(vocab, testpref, outprefix, lang)

                make_dataset(vocab, testpref, outprefix, lang, num_workers=args.workers)



    make_all(args.source_lang, src_dict)

    if target: