Commit 38f1dee9 authored by Davide Caroselli's avatar Davide Caroselli Committed by Facebook Github Bot
Browse files

Enforce UTF-8 when open() text files (#460) 

Summary:
When opening text files without specifying the encoding (i.e. `open(path, "r")` or `open(path, "w")`), python3 will use the preferred locale encoding (`locale.getpreferredencoding()`) so the result is platform dependent and can change from one machine to another.

I believe fairseq should enforce its standard (UTF-8 seems like the best choice to me). This pull request explicity specify UTF-8 encoding when reading text files.
Pull Request resolved: https://github.com/pytorch/fairseq/pull/460

Differential Revision: D13802525

Pulled By: myleott

fbshipit-source-id: 672fd55707ee559ab36d74bc1c24026166ea2367
parent ef3e6ab5

fairseq/data/indexed_dataset.py

+1 −1
Original line number Diff line number Diff line
@@ -168,7 +168,7 @@ class IndexedRawTextDataset(torch.utils.data.Dataset): 
        self.size = len(self.tokens_list)



    def read_data(self, path, dictionary):

        with open(path, 'r') as f:

        with open(path, 'r', encoding='utf-8') as f:

            for line in f:

                self.lines.append(line.strip('\n'))

                tokens = Tokenizer.tokenize(

fairseq/tokenizer.py

+3 −3
Original line number Diff line number Diff line
@@ -36,7 +36,7 @@ class Tokenizer: 
    @staticmethod

    def add_file_to_dictionary_single_worker(filename, tokenize, eos_word, worker_id=0, num_workers=1):

        counter = Counter()

        with open(filename, 'r') as f:

        with open(filename, 'r', encoding='utf-8') as f:

            size = os.fstat(f.fileno()).st_size

            chunk_size = size // num_workers

            offset = worker_id * chunk_size
@@ -86,7 +86,7 @@ class Tokenizer: 
            if idx == dict.unk_index and word != dict.unk_word:

                replaced.update([word])



        with open(filename, 'r') as f:

        with open(filename, 'r', encoding='utf-8') as f:

            f.seek(offset)

            # next(f) breaks f.tell(), hence readline() must be used

            line = safe_readline(f)
@@ -110,7 +110,7 @@ class Tokenizer: 


    @staticmethod

    def find_offsets(filename, num_chunks):

        with open(filename, 'r') as f:

        with open(filename, 'r', encoding='utf-8') as f:

            size = os.fstat(f.fileno()).st_size

            chunk_size = size // num_chunks

            offsets = [0 for _ in range(num_chunks + 1)]

preprocess.py

+4 −4
Original line number Diff line number Diff line
@@ -232,9 +232,9 @@ def main(args): 
        src_dict = dictionary.Dictionary.load(dict_path(args.source_lang))

        tgt_dict = dictionary.Dictionary.load(dict_path(args.target_lang))

        freq_map = {}

        with open(args.alignfile, "r") as align_file:

            with open(src_file_name, "r") as src_file:

                with open(tgt_file_name, "r") as tgt_file:

        with open(args.alignfile, "r", encoding='utf-8') as align_file:

            with open(src_file_name, "r", encoding='utf-8') as src_file:

                with open(tgt_file_name, "r", encoding='utf-8') as tgt_file:

                    for a, s, t in zip_longest(align_file, src_file, tgt_file):

                        si = Tokenizer.tokenize(s, src_dict, add_if_not_exist=False)

                        ti = Tokenizer.tokenize(t, tgt_dict, add_if_not_exist=False)
@@ -264,7 +264,7 @@ def main(args): 
                args.destdir,

                "alignment.{}-{}.txt".format(args.source_lang, args.target_lang),

            ),

            "w",

            "w", encoding='utf-8'

        ) as f:

            for k, v in align_dict.items():

                print("{} {}".format(src_dict[k], tgt_dict[v]), file=f)

scripts/build_sym_alignment.py

+2 −2
Original line number Diff line number Diff line
@@ -59,8 +59,8 @@ def main(): 


    # create joined file

    joined_file = os.path.join(args.output_dir, 'text.joined')

    with open(args.source_file, 'r') as src, open(args.target_file, 'r') as tgt:

        with open(joined_file, 'w') as joined:

    with open(args.source_file, 'r', encoding='utf-8') as src, open(args.target_file, 'r', encoding='utf-8') as tgt:

        with open(joined_file, 'w', encoding='utf-8') as joined:

            for s, t in zip_longest(src, tgt):

                print('{} ||| {}'.format(s.strip(), t.strip()), file=joined)