Loading probing.py +18 −0 Changes for probing.py: 18 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -17,11 +17,16 @@ sentences2_nlp = [nlp(sent) for sent in sentences2] dependency_hierarchy1 = [] dependency_hierarchy2 = [] sent1roots = [] sent2roots= [] for sent1 in sentences1_nlp: sentence_tokens = [token for token in sent1] dep_hierarchy1 = [] dep_hierarchy1.append([token for token in sent1 if token.dep_ == "ROOT"]) sent1roots.append([token.text for token in sent1 if token.dep_ == "ROOT"][0]) sentence_tokens.remove(dep_hierarchy1[0][0]) while len(sentence_tokens) != 0: Loading @@ -41,6 +46,7 @@ for sent2 in sentences2_nlp: sentence_tokens = [token for token in sent2] dep_hierarchy2 = [] dep_hierarchy2.append([token for token in sent2 if token.dep_ == "ROOT"]) sent2roots.append([token.text for token in sent2 if token.dep_ == "ROOT"][0]) sentence_tokens.remove(dep_hierarchy2[0][0]) while len(sentence_tokens) != 0: Loading Loading @@ -140,3 +146,15 @@ def mask_first_token(): return df def mask_root(): df = STSB_Dataset('data/stsbenchmark/sts-test.csv').as_dataframe for i in range(len(sent1roots)): df.iloc[i, 0] = df.iloc[i, 0].replace(sent1roots[i], get_random_string(random.randrange(1, 9))) for e in range(len(sent2roots)): df.iloc[e, 1] = df.iloc[e, 1].replace(sent2roots[2], get_random_string(random.randrange(1, 9))) return df Loading
probing.py +18 −0 Changes for probing.py: 18 added lines, 0 removed lines. Original line number Diff line number Diff line Loading @@ -17,11 +17,16 @@ sentences2_nlp = [nlp(sent) for sent in sentences2] dependency_hierarchy1 = [] dependency_hierarchy2 = [] sent1roots = [] sent2roots= [] for sent1 in sentences1_nlp: sentence_tokens = [token for token in sent1] dep_hierarchy1 = [] dep_hierarchy1.append([token for token in sent1 if token.dep_ == "ROOT"]) sent1roots.append([token.text for token in sent1 if token.dep_ == "ROOT"][0]) sentence_tokens.remove(dep_hierarchy1[0][0]) while len(sentence_tokens) != 0: Loading @@ -41,6 +46,7 @@ for sent2 in sentences2_nlp: sentence_tokens = [token for token in sent2] dep_hierarchy2 = [] dep_hierarchy2.append([token for token in sent2 if token.dep_ == "ROOT"]) sent2roots.append([token.text for token in sent2 if token.dep_ == "ROOT"][0]) sentence_tokens.remove(dep_hierarchy2[0][0]) while len(sentence_tokens) != 0: Loading Loading @@ -140,3 +146,15 @@ def mask_first_token(): return df def mask_root(): df = STSB_Dataset('data/stsbenchmark/sts-test.csv').as_dataframe for i in range(len(sent1roots)): df.iloc[i, 0] = df.iloc[i, 0].replace(sent1roots[i], get_random_string(random.randrange(1, 9))) for e in range(len(sent2roots)): df.iloc[e, 1] = df.iloc[e, 1].replace(sent2roots[2], get_random_string(random.randrange(1, 9))) return df