Loading sentence_similarity.py +41 −34 Changes for sentence_similarity.py: 41 added lines, 34 removed lines. Original line number Diff line number Diff line Loading @@ -9,51 +9,58 @@ from runLIT import LIT_STSB_Dataset from scipy.stats import spearmanr, pearsonr from sklearn import metrics dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv') zero_model = SentenceTransformer('models/nli-bert-large/') ft_model = SentenceTransformer('models/stsb-bert-large/') sentences1 = [s['sentence1'] for s in dataset._examples] sentences2 = [s['sentence2'] for s in dataset._examples] labels = [s['label'] for s in dataset._examples] #Compute embedding for both lists for zero model embeddings1 = zero_model.encode(sentences1, convert_to_tensor=True) embeddings2 = zero_model.encode(sentences2, convert_to_tensor=True) class SBERT_Model: def __init__(self, name, filepath, dataset): self.name = name self.filepath = filepath self.dataset = dataset self.model = SentenceTransformer(filepath) self.sentences1 = [s['sentence1'] for s in self.dataset._examples] self.sentences2 = [s['sentence2'] for s in self.dataset._examples] self.labels = [s['label'] for s in self.dataset._examples] self.embeddings1 = self.get_embeddings(self.sentences1) self.embeddings2 = self.get_embeddings(self.sentences2) self.cosine_scores = self.get_cosine_scores() self.preds = self.get_preds() def get_embeddings(self, sentences): return self.model.encode(sentences, convert_to_tensor=True) def get_preds(self): """This method extracts the scores for similarity between sentence pairs Cosine_scores have scores for similarity between all of the sentences, but we only need similarity between each sentence pair. """ preds = [] for i in range(len(self.cosine_scores[0])): preds.append(self.cosine_scores[i][i]) return preds #Compute embedding for both lists for fine-tuned model embeddings1ft = ft_model.encode(sentences1, convert_to_tensor=True) embeddings2ft = ft_model.encode(sentences2, convert_to_tensor=True) def get_cosine_scores(self): return util.pytorch_cos_sim(self.embeddings1, self.embeddings2) #Compute cosine-similarits cosine_scores = util.pytorch_cos_sim(embeddings1, embeddings2) cosine_scores_ft = util.pytorch_cos_sim(embeddings1ft, embeddings2ft) def get_pearson(self): return pearsonr(self.labels, self.preds)[0] # Save only the cosine scores of i-th pairs preds = [] for i in range(len(cosine_scores[0])): preds.append(cosine_scores[i][i]) def get_spearman(self): return spearmanr(self.labels, self.preds)[0] def get_MSE(self): return metrics.mean_squared_error(self.labels, self.preds) preds_ft = [] for i in range(len(cosine_scores_ft[0])): preds_ft.append(cosine_scores_ft[i][i]) def print_statistics(self): print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}") mse_zero = metrics.mean_squared_error(labels, preds) pearsonr_zero = pearsonr(labels, preds)[0] spearmanr_zero = spearmanr(labels, preds)[0] dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv') mse_ft = metrics.mean_squared_error(labels, preds_ft) pearsonr_ft = pearsonr(labels, preds_ft)[0] spearmanr_ft = spearmanr(labels, preds_ft)[0] zero_model = SBERT_Model("Zero Model", 'models/nli-bert-large/', dataset) ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset) zero_model.print_statistics() ft_model.print_statistics() print(f"Zero Model: MSE:{mse_zero}; Pearson:{pearsonr_zero}; Spearman:{spearmanr_zero})") print(f"Fine-tuned Model: MSE:{mse_ft}; Pearson:{pearsonr_ft}; Spearman:{spearmanr_ft})") #Output the pairs with their score Loading Loading
sentence_similarity.py +41 −34 Changes for sentence_similarity.py: 41 added lines, 34 removed lines. Original line number Diff line number Diff line Loading @@ -9,51 +9,58 @@ from runLIT import LIT_STSB_Dataset from scipy.stats import spearmanr, pearsonr from sklearn import metrics dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv') zero_model = SentenceTransformer('models/nli-bert-large/') ft_model = SentenceTransformer('models/stsb-bert-large/') sentences1 = [s['sentence1'] for s in dataset._examples] sentences2 = [s['sentence2'] for s in dataset._examples] labels = [s['label'] for s in dataset._examples] #Compute embedding for both lists for zero model embeddings1 = zero_model.encode(sentences1, convert_to_tensor=True) embeddings2 = zero_model.encode(sentences2, convert_to_tensor=True) class SBERT_Model: def __init__(self, name, filepath, dataset): self.name = name self.filepath = filepath self.dataset = dataset self.model = SentenceTransformer(filepath) self.sentences1 = [s['sentence1'] for s in self.dataset._examples] self.sentences2 = [s['sentence2'] for s in self.dataset._examples] self.labels = [s['label'] for s in self.dataset._examples] self.embeddings1 = self.get_embeddings(self.sentences1) self.embeddings2 = self.get_embeddings(self.sentences2) self.cosine_scores = self.get_cosine_scores() self.preds = self.get_preds() def get_embeddings(self, sentences): return self.model.encode(sentences, convert_to_tensor=True) def get_preds(self): """This method extracts the scores for similarity between sentence pairs Cosine_scores have scores for similarity between all of the sentences, but we only need similarity between each sentence pair. """ preds = [] for i in range(len(self.cosine_scores[0])): preds.append(self.cosine_scores[i][i]) return preds #Compute embedding for both lists for fine-tuned model embeddings1ft = ft_model.encode(sentences1, convert_to_tensor=True) embeddings2ft = ft_model.encode(sentences2, convert_to_tensor=True) def get_cosine_scores(self): return util.pytorch_cos_sim(self.embeddings1, self.embeddings2) #Compute cosine-similarits cosine_scores = util.pytorch_cos_sim(embeddings1, embeddings2) cosine_scores_ft = util.pytorch_cos_sim(embeddings1ft, embeddings2ft) def get_pearson(self): return pearsonr(self.labels, self.preds)[0] # Save only the cosine scores of i-th pairs preds = [] for i in range(len(cosine_scores[0])): preds.append(cosine_scores[i][i]) def get_spearman(self): return spearmanr(self.labels, self.preds)[0] def get_MSE(self): return metrics.mean_squared_error(self.labels, self.preds) preds_ft = [] for i in range(len(cosine_scores_ft[0])): preds_ft.append(cosine_scores_ft[i][i]) def print_statistics(self): print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}") mse_zero = metrics.mean_squared_error(labels, preds) pearsonr_zero = pearsonr(labels, preds)[0] spearmanr_zero = spearmanr(labels, preds)[0] dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv') mse_ft = metrics.mean_squared_error(labels, preds_ft) pearsonr_ft = pearsonr(labels, preds_ft)[0] spearmanr_ft = spearmanr(labels, preds_ft)[0] zero_model = SBERT_Model("Zero Model", 'models/nli-bert-large/', dataset) ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset) zero_model.print_statistics() ft_model.print_statistics() print(f"Zero Model: MSE:{mse_zero}; Pearson:{pearsonr_zero}; Spearman:{spearmanr_zero})") print(f"Fine-tuned Model: MSE:{mse_ft}; Pearson:{pearsonr_ft}; Spearman:{spearmanr_ft})") #Output the pairs with their score Loading