Commit 9c9b8b1e authored by podrazka's avatar podrazka
Browse files

Put SBERT_Model class and STSB_Dataset class in separate files

parent dae61ee8
Loading
Loading
Loading
Loading

SBERT_Model.py

0 → 100644
+49 −0
Changes for SBERT_Model.py: 49 added lines, 0 removed lines.
Original line number Diff line number Diff line
from sentence_transformers import SentenceTransformer, util

import STSB_Dataset

from scipy.stats import spearmanr, pearsonr
from sklearn import metrics

class SBERT_Model:
    def __init__(self, name, filepath, dataset, probing=False):
        self.name = name
        self.filepath = filepath
        self.dataset = dataset
        self.model = SentenceTransformer(filepath)
        self.sentences1 = [s['sentence1'] for s in self.dataset._examples]
        self.sentences2 = [s['sentence2'] for s in self.dataset._examples]
    
        self.labels = [s['label'] for s in self.dataset._examples]
        self.embeddings1 = self.get_embeddings(self.sentences1)
        self.embeddings2 = self.get_embeddings(self.sentences2)
        self.cosine_scores = self.get_cosine_scores()
        self.preds = self.get_preds()

    def get_embeddings(self, sentences):
        return self.model.encode(sentences, convert_to_tensor=True)

    def get_preds(self):
        """This method extracts the scores for similarity between sentence pairs
        Cosine_scores have scores for similarity between all of the sentences, 
        but we only need similarity between each sentence pair.
        """
        preds = []
        for i in range(len(self.cosine_scores[0])):
            preds.append(self.cosine_scores[i][i])
        return preds

    def get_cosine_scores(self):
        return util.pytorch_cos_sim(self.embeddings1, self.embeddings2)

    def get_pearson(self):
        return pearsonr(self.labels, self.preds)[0]

    def get_spearman(self):
        return spearmanr(self.labels, self.preds)[0]

    def get_MSE(self):
        return metrics.mean_squared_error(self.labels, self.preds)
    
    def print_statistics(self):
        print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}")
 No newline at end of file

STSB_Dataset.py

0 → 100644
+41 −0
Changes for STSB_Dataset.py: 41 added lines, 0 removed lines.
Original line number Diff line number Diff line
from lit_nlp.api import dataset as lit_dataset
from lit_nlp.api import types as lit_types

class STSB_Dataset(lit_dataset.Dataset):
    
    # this is a LIT-specific wrapper for our STS-B Dataset.
    
    def __init__(self, path):

        with open(path,
                  'r', encoding='utf-8') as d:
            dataset = d.read().splitlines()

        self.data = []
        self.labels = []


        for line in dataset:
            datapoint = line.split('\t')
            score = float(datapoint[4])
            score = score/5

            st1 = datapoint[5]
            st2 = datapoint[6]
            self.data.append([st1,st2])
            self.labels.append(score)

            
        # Store as a list of dicts, conforming to self.spec()
        self._examples = [{
          'sentence1': dp[0],
          'sentence2': dp[1],
          'label': label,
        } for dp, label in zip(self.data, self.labels)]

    def spec(self):
        return {
          'sentence1': lit_types.TextSegment(),
          'sentence2': lit_types.TextSegment(),
          'label': lit_types.RegressionScore(),
        }
 No newline at end of file
+3 −60
Changes for sentence_similarity.py: 3 added lines, 60 removed lines.
Original line number Diff line number Diff line
# Careful, if you install sentence_transformers on lit-nlp environment, 
# runLIT.py will not work because it uses an old version of transformers (2.11.0). 
# Make a clone of the environment instead: conda create --name myclone --clone lit-nlp
from SBERT_Model import SBERT_Model
from STSB_Dataset import STSB_Dataset

from sentence_transformers import SentenceTransformer, util

from runLIT import LIT_STSB_Dataset

from scipy.stats import spearmanr, pearsonr
from sklearn import metrics

class SBERT_Model:
    def __init__(self, name, filepath, dataset, probing=False):
        self.name = name
        self.filepath = filepath
        self.dataset = dataset
        self.model = SentenceTransformer(filepath)
        self.sentences1 = [s['sentence1'] for s in self.dataset._examples]
        self.sentences2 = [s['sentence2'] for s in self.dataset._examples]
    
        self.labels = [s['label'] for s in self.dataset._examples]
        self.embeddings1 = self.get_embeddings(self.sentences1)
        self.embeddings2 = self.get_embeddings(self.sentences2)
        self.cosine_scores = self.get_cosine_scores()
        self.preds = self.get_preds()

    def get_embeddings(self, sentences):
        return self.model.encode(sentences, convert_to_tensor=True)

    def get_preds(self):
        """This method extracts the scores for similarity between sentence pairs
        Cosine_scores have scores for similarity between all of the sentences, 
        but we only need similarity between each sentence pair.
        """
        preds = []
        for i in range(len(self.cosine_scores[0])):
            preds.append(self.cosine_scores[i][i])
        return preds

    def get_cosine_scores(self):
        return util.pytorch_cos_sim(self.embeddings1, self.embeddings2)

    def get_pearson(self):
        return pearsonr(self.labels, self.preds)[0]

    def get_spearman(self):
        return spearmanr(self.labels, self.preds)[0]

    def get_MSE(self):
        return metrics.mean_squared_error(self.labels, self.preds)
    
    def print_statistics(self):
        print(f"{self.name}: MSE:{self.get_MSE()}; Pearson:{self.get_pearson()}; Spearman:{self.get_spearman()}")


dataset = LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv')
dataset = STSB_Dataset('data/stsbenchmark/sts-test.csv')

zero_model = SBERT_Model("Zero Model", 'models/nli-bert-large/', dataset)
ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset)
@@ -61,10 +11,3 @@ ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset)
zero_model.print_statistics()
ft_model.print_statistics()



#Output the pairs with their score
#for i in range(len(sentences1)):
#    print(f"{sentences1[i]}; {sentences2[i]}; zero:{cosine_scores[i][i]}; fine-tuned:{cosine_scores_ft[i][i]}; gold:{labels[i]}")