Commit f6ac1aec authored by Sergey Edunov's avatar Sergey Edunov
Browse files

Force UTF-8 encoding for dictionary files ( #41 )

parent bb3be24d
Loading
Loading
Loading
Loading
+6 −2
Changes for fairseq/dictionary.py: 6 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -109,8 +109,12 @@ class Dictionary(object):
        """

        if isinstance(f, str):
            with open(f, 'r') as fd:
            try:
                with open(f, 'r', encoding='utf-8') as fd:
                    return Dictionary.load(fd)
            except:
                raise Exception("Incorrect encoding detected in {}, please "
                                "rebuild the dataset".format(f))

        d = Dictionary()
        for line in f.readlines():
@@ -125,7 +129,7 @@ class Dictionary(object):
    def save(self, f, threshold=3, nwords=-1):
        """Stores dictionary into a text file"""
        if isinstance(f, str):
            with open(f, 'w') as fd:
            with open(f, 'w', encoding='utf-8') as fd:
                return self.save(fd, threshold, nwords)
        cnt = 0
        for i, t in enumerate(zip(self.symbols, self.count)):