Loading fairseq/data.py +1 −1 Changes for fairseq/data.py: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -442,7 +442,7 @@ def numpy_seed(seed): def get_dummy_batch(ntokens, src_dict, dst_dict, src_len=128, tgt_len=128): bsz = int(ntokens / max(src_len, tgt_len)) bsz = (bsz // 8) * 8 bsz = math.ceil(bsz / 8) * 8 assert src_dict.pad() == dst_dict.pad() pad_idx = src_dict.pad() src_vocab, dst_vocab = len(src_dict), len(dst_dict) Loading fairseq/dictionary.py +2 −1 Changes for fairseq/dictionary.py: 2 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -93,9 +93,10 @@ class Dictionary(object): multiple of 8, which is important on some hardware (e.g., Nvidia Tensor Cores). """ if padding_factor > 1: if nwords == -1: nwords = len(self) if padding_factor > 1: i = 0 while nwords % padding_factor != 0: if nwords >= len(self): Loading scripts/average_checkpoints.py +4 −1 Changes for scripts/average_checkpoints.py: 4 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -44,7 +44,10 @@ def average_checkpoints(inputs): for k in params_keys: if k not in params_dict: params_dict[k] = [] params_dict[k].append(model_params[k].float()) p = model_params[k] if isinstance(p, torch.HalfTensor): p = p.float() params_dict[k].append(p) averaged_params = collections.OrderedDict() # v should be a list of torch Tensor. Loading tests/utils.py +1 −1 Changes for tests/utils.py: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -21,7 +21,7 @@ def dummy_dictionary(vocab_size, prefix='token_'): for i in range(vocab_size): token = prefix + str(i) d.add_symbol(token) d.finalize() d.finalize(padding_factor=1) # don't add extra padding symbols return d Loading Loading
fairseq/data.py +1 −1 Changes for fairseq/data.py: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -442,7 +442,7 @@ def numpy_seed(seed): def get_dummy_batch(ntokens, src_dict, dst_dict, src_len=128, tgt_len=128): bsz = int(ntokens / max(src_len, tgt_len)) bsz = (bsz // 8) * 8 bsz = math.ceil(bsz / 8) * 8 assert src_dict.pad() == dst_dict.pad() pad_idx = src_dict.pad() src_vocab, dst_vocab = len(src_dict), len(dst_dict) Loading
fairseq/dictionary.py +2 −1 Changes for fairseq/dictionary.py: 2 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -93,9 +93,10 @@ class Dictionary(object): multiple of 8, which is important on some hardware (e.g., Nvidia Tensor Cores). """ if padding_factor > 1: if nwords == -1: nwords = len(self) if padding_factor > 1: i = 0 while nwords % padding_factor != 0: if nwords >= len(self): Loading
scripts/average_checkpoints.py +4 −1 Changes for scripts/average_checkpoints.py: 4 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -44,7 +44,10 @@ def average_checkpoints(inputs): for k in params_keys: if k not in params_dict: params_dict[k] = [] params_dict[k].append(model_params[k].float()) p = model_params[k] if isinstance(p, torch.HalfTensor): p = p.float() params_dict[k].append(p) averaged_params = collections.OrderedDict() # v should be a list of torch Tensor. Loading
tests/utils.py +1 −1 Changes for tests/utils.py: 1 added line, 1 removed line. Original line number Diff line number Diff line Loading @@ -21,7 +21,7 @@ def dummy_dictionary(vocab_size, prefix='token_'): for i in range(vocab_size): token = prefix + str(i) d.add_symbol(token) d.finalize() d.finalize(padding_factor=1) # don't add extra padding symbols return d Loading