Commit a2855929 authored by podrazka's avatar podrazka
Browse files

Add the calculation of MSE, Pearson, Spearman correlation statistics

parent d4853d26
Loading
Loading
Loading
Loading
+45 −10
Changes for sentence_similarity.py: 45 added lines, 10 removed lines.
Original line number Diff line number Diff line
@@ -6,22 +6,57 @@ from sentence_transformers import SentenceTransformer, util

from runLIT import LIT_STSB_Dataset

from scipy.stats import spearmanr, pearsonr
from sklearn import metrics

dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv')

model = SentenceTransformer('models/stsb-bert-large/')
zero_model = SentenceTransformer('models/nli-bert-large/')
ft_model = SentenceTransformer('models/stsb-bert-large/')

sentences1 = [s['sentence1'] for s in dataset._examples]
sentences2 = [s['sentence2'] for s in dataset._examples]
labels = [s['label'] for s in dataset._examples]

# Demo on the first 50 elements
sentences1 = [s['sentence1'] for s in dataset._examples][:50]
sentences2 = [s['sentence2'] for s in dataset._examples][:50]
labels = [s['label'] for s in dataset._examples][:50]
#Compute embedding for both lists for zero model
embeddings1 = zero_model.encode(sentences1, convert_to_tensor=True)
embeddings2 = zero_model.encode(sentences2, convert_to_tensor=True)

#Compute embedding for both lists
embeddings1 = model.encode(sentences1, convert_to_tensor=True)
embeddings2 = model.encode(sentences2, convert_to_tensor=True)

#Compute embedding for both lists for fine-tuned model
embeddings1ft = ft_model.encode(sentences1, convert_to_tensor=True)
embeddings2ft = ft_model.encode(sentences2, convert_to_tensor=True)

#Compute cosine-similarits
cosine_scores = util.pytorch_cos_sim(embeddings1, embeddings2)
cosine_scores_ft = util.pytorch_cos_sim(embeddings1ft, embeddings2ft)

# Save only the cosine scores of i-th pairs
preds = []
for i in range(len(cosine_scores[0])):
    preds.append(cosine_scores[i][i])


preds_ft = []
for i in range(len(cosine_scores_ft[0])):
    preds_ft.append(cosine_scores_ft[i][i])


mse_zero = metrics.mean_squared_error(labels, preds)
pearsonr_zero = pearsonr(labels, preds)[0]
spearmanr_zero = spearmanr(labels, preds)[0]

mse_ft = metrics.mean_squared_error(labels, preds_ft)
pearsonr_ft = pearsonr(labels, preds_ft)[0]
spearmanr_ft = spearmanr(labels, preds_ft)[0]


print(f"Zero Model: MSE:{mse_zero}; Pearson:{pearsonr_zero}; Spearman:{spearmanr_zero})")

print(f"Fine-tuned Model: MSE:{mse_ft}; Pearson:{pearsonr_ft}; Spearman:{spearmanr_ft})")


#Output the pairs with their score
for i in range(len(sentences1)):
    print("{} \t\t {} \t\t Score: {:.4f} Label: {:.4f}".format(sentences1[i], sentences2[i], cosine_scores[i][i], labels[i]))
 No newline at end of file
#for i in range(len(sentences1)):
#    print(f"{sentences1[i]}; {sentences2[i]}; zero:{cosine_scores[i][i]}; fine-tuned:{cosine_scores_ft[i][i]}; gold:{labels[i]}")