Loading SBERT_Model.py 0 → 100644 +49 −0 Changes for SBERT_Model.py: 49 added lines, 0 removed lines. Original line number Diff line number Diff line from sentence_transformers import SentenceTransformer, util import STSB_Dataset from scipy.stats import spearmanr, pearsonr from sklearn import metrics class SBERT_Model: def __init__(self, name, filepath, dataset, probing=False): self.name = name self.filepath = filepath self.dataset = dataset self.model = SentenceTransformer(filepath) self.sentences1 = [s['sentence1'] for s in self.dataset._examples] self.sentences2 = [s['sentence2'] for s in self.dataset._examples] self.labels = [s['label'] for s in self.dataset._examples] self.embeddings1 = self.get_embeddings(self.sentences1) self.embeddings2 = self.get_embeddings(self.sentences2) self.cosine_scores = self.get_cosine_scores() self.preds = self.get_preds() def get_embeddings(self, sentences): return self.model.encode(sentences, convert_to_tensor=True) def get_preds(self): """This method extracts the scores for similarity between sentence pairs Cosine_scores have scores for similarity between all of the sentences, but we only need similarity between each sentence pair. """ preds = [] for i in range(len(self.cosine_scores[0])): preds.append(self.cosine_scores[i][i]) return preds def get_cosine_scores(self): return util.pytorch_cos_sim(self.embeddings1, self.embeddings2) def get_pearson(self): return pearsonr(self.labels, self.preds)[0] def get_spearman(self): return spearmanr(self.labels, self.preds)[0] def get_MSE(self): return metrics.mean_squared_error(self.labels, self.preds) def print_statistics(self): print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}") No newline at end of file STSB_Dataset.py 0 → 100644 +41 −0 Changes for STSB_Dataset.py: 41 added lines, 0 removed lines. Original line number Diff line number Diff line from lit_nlp.api import dataset as lit_dataset from lit_nlp.api import types as lit_types class STSB_Dataset(lit_dataset.Dataset): # this is a LIT-specific wrapper for our STS-B Dataset. def __init__(self, path): with open(path, 'r', encoding='utf-8') as d: dataset = d.read().splitlines() self.data = [] self.labels = [] for line in dataset: datapoint = line.split('\t') score = float(datapoint[4]) score = score/5 st1 = datapoint[5] st2 = datapoint[6] self.data.append([st1,st2]) self.labels.append(score) # Store as a list of dicts, conforming to self.spec() self._examples = [{ 'sentence1': dp[0], 'sentence2': dp[1], 'label': label, } for dp, label in zip(self.data, self.labels)] def spec(self): return { 'sentence1': lit_types.TextSegment(), 'sentence2': lit_types.TextSegment(), 'label': lit_types.RegressionScore(), } No newline at end of file sentence_similarity.py +3 −60 Changes for sentence_similarity.py: 3 added lines, 60 removed lines. Original line number Diff line number Diff line # Careful, if you install sentence_transformers on lit-nlp environment, # runLIT.py will not work because it uses an old version of transformers (2.11.0). # Make a clone of the environment instead: conda create --name myclone --clone lit-nlp from SBERT_Model import SBERT_Model from STSB_Dataset import STSB_Dataset from sentence_transformers import SentenceTransformer, util from runLIT import LIT_STSB_Dataset from scipy.stats import spearmanr, pearsonr from sklearn import metrics class SBERT_Model: def __init__(self, name, filepath, dataset, probing=False): self.name = name self.filepath = filepath self.dataset = dataset self.model = SentenceTransformer(filepath) self.sentences1 = [s['sentence1'] for s in self.dataset._examples] self.sentences2 = [s['sentence2'] for s in self.dataset._examples] self.labels = [s['label'] for s in self.dataset._examples] self.embeddings1 = self.get_embeddings(self.sentences1) self.embeddings2 = self.get_embeddings(self.sentences2) self.cosine_scores = self.get_cosine_scores() self.preds = self.get_preds() def get_embeddings(self, sentences): return self.model.encode(sentences, convert_to_tensor=True) def get_preds(self): """This method extracts the scores for similarity between sentence pairs Cosine_scores have scores for similarity between all of the sentences, but we only need similarity between each sentence pair. """ preds = [] for i in range(len(self.cosine_scores[0])): preds.append(self.cosine_scores[i][i]) return preds def get_cosine_scores(self): return util.pytorch_cos_sim(self.embeddings1, self.embeddings2) def get_pearson(self): return pearsonr(self.labels, self.preds)[0] def get_spearman(self): return spearmanr(self.labels, self.preds)[0] def get_MSE(self): return metrics.mean_squared_error(self.labels, self.preds) def print_statistics(self): print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}") dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv') dataset = STSB_Dataset('data/stsbenchmark/sts-test.csv') zero_model = SBERT_Model("Zero Model", 'models/nli-bert-large/', dataset) ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset) Loading @@ -61,10 +11,3 @@ ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset) zero_model.print_statistics() ft_model.print_statistics() #Output the pairs with their score #for i in range(len(sentences1)): # print(f"{sentences1[i]}; {sentences2[i]}; zero:{cosine_scores[i][i]}; fine-tuned:{cosine_scores_ft[i][i]}; gold:{labels[i]}") Loading
SBERT_Model.py 0 → 100644 +49 −0 Changes for SBERT_Model.py: 49 added lines, 0 removed lines. Original line number Diff line number Diff line from sentence_transformers import SentenceTransformer, util import STSB_Dataset from scipy.stats import spearmanr, pearsonr from sklearn import metrics class SBERT_Model: def __init__(self, name, filepath, dataset, probing=False): self.name = name self.filepath = filepath self.dataset = dataset self.model = SentenceTransformer(filepath) self.sentences1 = [s['sentence1'] for s in self.dataset._examples] self.sentences2 = [s['sentence2'] for s in self.dataset._examples] self.labels = [s['label'] for s in self.dataset._examples] self.embeddings1 = self.get_embeddings(self.sentences1) self.embeddings2 = self.get_embeddings(self.sentences2) self.cosine_scores = self.get_cosine_scores() self.preds = self.get_preds() def get_embeddings(self, sentences): return self.model.encode(sentences, convert_to_tensor=True) def get_preds(self): """This method extracts the scores for similarity between sentence pairs Cosine_scores have scores for similarity between all of the sentences, but we only need similarity between each sentence pair. """ preds = [] for i in range(len(self.cosine_scores[0])): preds.append(self.cosine_scores[i][i]) return preds def get_cosine_scores(self): return util.pytorch_cos_sim(self.embeddings1, self.embeddings2) def get_pearson(self): return pearsonr(self.labels, self.preds)[0] def get_spearman(self): return spearmanr(self.labels, self.preds)[0] def get_MSE(self): return metrics.mean_squared_error(self.labels, self.preds) def print_statistics(self): print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}") No newline at end of file
STSB_Dataset.py 0 → 100644 +41 −0 Changes for STSB_Dataset.py: 41 added lines, 0 removed lines. Original line number Diff line number Diff line from lit_nlp.api import dataset as lit_dataset from lit_nlp.api import types as lit_types class STSB_Dataset(lit_dataset.Dataset): # this is a LIT-specific wrapper for our STS-B Dataset. def __init__(self, path): with open(path, 'r', encoding='utf-8') as d: dataset = d.read().splitlines() self.data = [] self.labels = [] for line in dataset: datapoint = line.split('\t') score = float(datapoint[4]) score = score/5 st1 = datapoint[5] st2 = datapoint[6] self.data.append([st1,st2]) self.labels.append(score) # Store as a list of dicts, conforming to self.spec() self._examples = [{ 'sentence1': dp[0], 'sentence2': dp[1], 'label': label, } for dp, label in zip(self.data, self.labels)] def spec(self): return { 'sentence1': lit_types.TextSegment(), 'sentence2': lit_types.TextSegment(), 'label': lit_types.RegressionScore(), } No newline at end of file
sentence_similarity.py +3 −60 Changes for sentence_similarity.py: 3 added lines, 60 removed lines. Original line number Diff line number Diff line # Careful, if you install sentence_transformers on lit-nlp environment, # runLIT.py will not work because it uses an old version of transformers (2.11.0). # Make a clone of the environment instead: conda create --name myclone --clone lit-nlp from SBERT_Model import SBERT_Model from STSB_Dataset import STSB_Dataset from sentence_transformers import SentenceTransformer, util from runLIT import LIT_STSB_Dataset from scipy.stats import spearmanr, pearsonr from sklearn import metrics class SBERT_Model: def __init__(self, name, filepath, dataset, probing=False): self.name = name self.filepath = filepath self.dataset = dataset self.model = SentenceTransformer(filepath) self.sentences1 = [s['sentence1'] for s in self.dataset._examples] self.sentences2 = [s['sentence2'] for s in self.dataset._examples] self.labels = [s['label'] for s in self.dataset._examples] self.embeddings1 = self.get_embeddings(self.sentences1) self.embeddings2 = self.get_embeddings(self.sentences2) self.cosine_scores = self.get_cosine_scores() self.preds = self.get_preds() def get_embeddings(self, sentences): return self.model.encode(sentences, convert_to_tensor=True) def get_preds(self): """This method extracts the scores for similarity between sentence pairs Cosine_scores have scores for similarity between all of the sentences, but we only need similarity between each sentence pair. """ preds = [] for i in range(len(self.cosine_scores[0])): preds.append(self.cosine_scores[i][i]) return preds def get_cosine_scores(self): return util.pytorch_cos_sim(self.embeddings1, self.embeddings2) def get_pearson(self): return pearsonr(self.labels, self.preds)[0] def get_spearman(self): return spearmanr(self.labels, self.preds)[0] def get_MSE(self): return metrics.mean_squared_error(self.labels, self.preds) def print_statistics(self): print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}") dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv') dataset = STSB_Dataset('data/stsbenchmark/sts-test.csv') zero_model = SBERT_Model("Zero Model", 'models/nli-bert-large/', dataset) ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset) Loading @@ -61,10 +11,3 @@ ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset) zero_model.print_statistics() ft_model.print_statistics() #Output the pairs with their score #for i in range(len(sentences1)): # print(f"{sentences1[i]}; {sentences2[i]}; zero:{cosine_scores[i][i]}; fine-tuned:{cosine_scores_ft[i][i]}; gold:{labels[i]}")