Loading sentence_similarity.py +45 −10 Changes for sentence_similarity.py: 45 added lines, 10 removed lines. Original line number Diff line number Diff line Loading @@ -6,22 +6,57 @@ from sentence_transformers import SentenceTransformer, util from runLIT import LIT_STSB_Dataset from scipy.stats import spearmanr, pearsonr from sklearn import metrics dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv') model = SentenceTransformer('models/stsb-bert-large/') zero_model = SentenceTransformer('models/nli-bert-large/') ft_model = SentenceTransformer('models/stsb-bert-large/') sentences1 = [s['sentence1'] for s in dataset._examples] sentences2 = [s['sentence2'] for s in dataset._examples] labels = [s['label'] for s in dataset._examples] # Demo on the first 50 elements sentences1 = [s['sentence1'] for s in dataset._examples][:50] sentences2 = [s['sentence2'] for s in dataset._examples][:50] labels = [s['label'] for s in dataset._examples][:50] #Compute embedding for both lists for zero model embeddings1 = zero_model.encode(sentences1, convert_to_tensor=True) embeddings2 = zero_model.encode(sentences2, convert_to_tensor=True) #Compute embedding for both lists embeddings1 = model.encode(sentences1, convert_to_tensor=True) embeddings2 = model.encode(sentences2, convert_to_tensor=True) #Compute embedding for both lists for fine-tuned model embeddings1ft = ft_model.encode(sentences1, convert_to_tensor=True) embeddings2ft = ft_model.encode(sentences2, convert_to_tensor=True) #Compute cosine-similarits cosine_scores = util.pytorch_cos_sim(embeddings1, embeddings2) cosine_scores_ft = util.pytorch_cos_sim(embeddings1ft, embeddings2ft) # Save only the cosine scores of i-th pairs preds = [] for i in range(len(cosine_scores[0])): preds.append(cosine_scores[i][i]) preds_ft = [] for i in range(len(cosine_scores_ft[0])): preds_ft.append(cosine_scores_ft[i][i]) mse_zero = metrics.mean_squared_error(labels, preds) pearsonr_zero = pearsonr(labels, preds)[0] spearmanr_zero = spearmanr(labels, preds)[0] mse_ft = metrics.mean_squared_error(labels, preds_ft) pearsonr_ft = pearsonr(labels, preds_ft)[0] spearmanr_ft = spearmanr(labels, preds_ft)[0] print(f"Zero Model: MSE:{mse_zero}; Pearson:{pearsonr_zero}; Spearman:{spearmanr_zero})") print(f"Fine-tuned Model: MSE:{mse_ft}; Pearson:{pearsonr_ft}; Spearman:{spearmanr_ft})") #Output the pairs with their score for i in range(len(sentences1)): print("{} \t\t {} \t\t Score: {:.4f} Label: {:.4f}".format(sentences1[i], sentences2[i], cosine_scores[i][i], labels[i])) No newline at end of file #for i in range(len(sentences1)): # print(f"{sentences1[i]}; {sentences2[i]}; zero:{cosine_scores[i][i]}; fine-tuned:{cosine_scores_ft[i][i]}; gold:{labels[i]}") Loading
sentence_similarity.py +45 −10 Changes for sentence_similarity.py: 45 added lines, 10 removed lines. Original line number Diff line number Diff line Loading @@ -6,22 +6,57 @@ from sentence_transformers import SentenceTransformer, util from runLIT import LIT_STSB_Dataset from scipy.stats import spearmanr, pearsonr from sklearn import metrics dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv') model = SentenceTransformer('models/stsb-bert-large/') zero_model = SentenceTransformer('models/nli-bert-large/') ft_model = SentenceTransformer('models/stsb-bert-large/') sentences1 = [s['sentence1'] for s in dataset._examples] sentences2 = [s['sentence2'] for s in dataset._examples] labels = [s['label'] for s in dataset._examples] # Demo on the first 50 elements sentences1 = [s['sentence1'] for s in dataset._examples][:50] sentences2 = [s['sentence2'] for s in dataset._examples][:50] labels = [s['label'] for s in dataset._examples][:50] #Compute embedding for both lists for zero model embeddings1 = zero_model.encode(sentences1, convert_to_tensor=True) embeddings2 = zero_model.encode(sentences2, convert_to_tensor=True) #Compute embedding for both lists embeddings1 = model.encode(sentences1, convert_to_tensor=True) embeddings2 = model.encode(sentences2, convert_to_tensor=True) #Compute embedding for both lists for fine-tuned model embeddings1ft = ft_model.encode(sentences1, convert_to_tensor=True) embeddings2ft = ft_model.encode(sentences2, convert_to_tensor=True) #Compute cosine-similarits cosine_scores = util.pytorch_cos_sim(embeddings1, embeddings2) cosine_scores_ft = util.pytorch_cos_sim(embeddings1ft, embeddings2ft) # Save only the cosine scores of i-th pairs preds = [] for i in range(len(cosine_scores[0])): preds.append(cosine_scores[i][i]) preds_ft = [] for i in range(len(cosine_scores_ft[0])): preds_ft.append(cosine_scores_ft[i][i]) mse_zero = metrics.mean_squared_error(labels, preds) pearsonr_zero = pearsonr(labels, preds)[0] spearmanr_zero = spearmanr(labels, preds)[0] mse_ft = metrics.mean_squared_error(labels, preds_ft) pearsonr_ft = pearsonr(labels, preds_ft)[0] spearmanr_ft = spearmanr(labels, preds_ft)[0] print(f"Zero Model: MSE:{mse_zero}; Pearson:{pearsonr_zero}; Spearman:{spearmanr_zero})") print(f"Fine-tuned Model: MSE:{mse_ft}; Pearson:{pearsonr_ft}; Spearman:{spearmanr_ft})") #Output the pairs with their score for i in range(len(sentences1)): print("{} \t\t {} \t\t Score: {:.4f} Label: {:.4f}".format(sentences1[i], sentences2[i], cosine_scores[i][i], labels[i])) No newline at end of file #for i in range(len(sentences1)): # print(f"{sentences1[i]}; {sentences2[i]}; zero:{cosine_scores[i][i]}; fine-tuned:{cosine_scores_ft[i][i]}; gold:{labels[i]}")