Commit 5e0553de authored by podrazka's avatar podrazka
Browse files

Add SBERT_Model class for better structure of storing model-related variables

parent a2855929
Loading
Loading
Loading
Loading
+41 −34
Changes for sentence_similarity.py: 41 added lines, 34 removed lines.
Original line number Diff line number Diff line
@@ -9,51 +9,58 @@ from runLIT import LIT_STSB_Dataset
from scipy.stats import spearmanr, pearsonr
from sklearn import metrics

dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv')

zero_model = SentenceTransformer('models/nli-bert-large/')
ft_model = SentenceTransformer('models/stsb-bert-large/')

sentences1 = [s['sentence1'] for s in dataset._examples]
sentences2 = [s['sentence2'] for s in dataset._examples]
labels = [s['label'] for s in dataset._examples]

#Compute embedding for both lists for zero model
embeddings1 = zero_model.encode(sentences1, convert_to_tensor=True)
embeddings2 = zero_model.encode(sentences2, convert_to_tensor=True)

class SBERT_Model:
    def __init__(self, name, filepath, dataset):
        self.name = name
        self.filepath = filepath
        self.dataset = dataset
        self.model = SentenceTransformer(filepath)
        self.sentences1 = [s['sentence1'] for s in self.dataset._examples]
        self.sentences2 = [s['sentence2'] for s in self.dataset._examples]
        self.labels = [s['label'] for s in self.dataset._examples]
        self.embeddings1 = self.get_embeddings(self.sentences1)
        self.embeddings2 = self.get_embeddings(self.sentences2)
        self.cosine_scores = self.get_cosine_scores()
        self.preds = self.get_preds()

    def get_embeddings(self, sentences):
        return self.model.encode(sentences, convert_to_tensor=True)

    def get_preds(self):
        """This method extracts the scores for similarity between sentence pairs
        Cosine_scores have scores for similarity between all of the sentences, 
        but we only need similarity between each sentence pair.
        """
        preds = []
        for i in range(len(self.cosine_scores[0])):
            preds.append(self.cosine_scores[i][i])
        return preds

#Compute embedding for both lists for fine-tuned model
embeddings1ft = ft_model.encode(sentences1, convert_to_tensor=True)
embeddings2ft = ft_model.encode(sentences2, convert_to_tensor=True)
    def get_cosine_scores(self):
        return util.pytorch_cos_sim(self.embeddings1, self.embeddings2)

#Compute cosine-similarits
cosine_scores = util.pytorch_cos_sim(embeddings1, embeddings2)
cosine_scores_ft = util.pytorch_cos_sim(embeddings1ft, embeddings2ft)
    def get_pearson(self):
        return pearsonr(self.labels, self.preds)[0]

# Save only the cosine scores of i-th pairs
preds = []
for i in range(len(cosine_scores[0])):
    preds.append(cosine_scores[i][i])
    def get_spearman(self):
        return spearmanr(self.labels, self.preds)[0]

    def get_MSE(self):
        return metrics.mean_squared_error(self.labels, self.preds)
    
preds_ft = []
for i in range(len(cosine_scores_ft[0])):
    preds_ft.append(cosine_scores_ft[i][i])
    def print_statistics(self):
        print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}")


mse_zero = metrics.mean_squared_error(labels, preds)
pearsonr_zero = pearsonr(labels, preds)[0]
spearmanr_zero = spearmanr(labels, preds)[0]
dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv')

mse_ft = metrics.mean_squared_error(labels, preds_ft)
pearsonr_ft = pearsonr(labels, preds_ft)[0]
spearmanr_ft = spearmanr(labels, preds_ft)[0]
zero_model = SBERT_Model("Zero Model", 'models/nli-bert-large/', dataset)
ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset)

zero_model.print_statistics()
ft_model.print_statistics()

print(f"Zero Model: MSE:{mse_zero}; Pearson:{pearsonr_zero}; Spearman:{spearmanr_zero})")

print(f"Fine-tuned Model: MSE:{mse_ft}; Pearson:{pearsonr_ft}; Spearman:{spearmanr_ft})")


#Output the pairs with their score