Commit d752d144 authored by podrazka's avatar podrazka
Browse files

Add a method for visualizing embeddings in 2D based on cosine metric. Also add...

Add a method for visualizing embeddings in 2D based on cosine metric. Also add a method for printing out clusters of highly similar sentences
parent bc0008f5
Loading
Loading
Loading
Loading
+89 −21
Changes for SBERT_Model.py: 89 added lines, 21 removed lines.
Original line number Diff line number Diff line
from sentence_transformers import SentenceTransformer, util

from scipy.stats import spearmanr, pearsonr
from sklearn import metrics
from sklearn.decomposition import PCA
from sklearn.cluster import AgglomerativeClustering
import umap
import matplotlib.pyplot as plt
from random import randrange
import numpy as np

import time
from datetime import datetime
import os


class SBERT_Model:
    def __init__(self, name, filepath, dataset, probing=False):
@@ -17,33 +23,95 @@ class SBERT_Model:
        self.model = SentenceTransformer(filepath)
        self.sentences1 = dataset['sentence1'].tolist()
        self.sentences2 = dataset['sentence2'].tolist()
    
        self.labels = dataset['label'].tolist()
        self.embeddings1 = self.get_embeddings(self.sentences1)
        self.embeddings2 = self.get_embeddings(self.sentences2)
        self.cosine_scores = self.get_cosine_scores()
        self.preds = self.get_preds()

    def visualize_embeddings(self):
        sentences = self.sentences1 + self.sentences2
    def sentence_clusters(self):
        """
        This method prints out clusters of sentences with high similarity.
        """
        corpus = self.sentences1 + self.sentences2
        corpus_embeddings = self.get_embeddings(corpus)

        # Normalize the embeddings to unit length
        corpus_embeddings = corpus_embeddings /  np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)

        # Perform kmean clustering
        clustering_model = AgglomerativeClustering(n_clusters=None, distance_threshold=1.5) #, affinity='cosine', linkage='average', distance_threshold=0.4)
        clustering_model.fit(corpus_embeddings)
        cluster_assignment = clustering_model.labels_

        clustered_sentences = {}
        for sentence_id, cluster_id in enumerate(cluster_assignment):
            if cluster_id not in clustered_sentences:
                clustered_sentences[cluster_id] = []

            clustered_sentences[cluster_id].append(corpus[sentence_id])

        for i, cluster in clustered_sentences.items():
            print("Cluster ", i+1)
            print(cluster)
            print("")

    def visualize_embeddings(self, sentences, highlights = [], figsize=(80,200)):
        """
        Parameters
        ----------
        sentences: list of strings
            list of sentences that will be visualized and colored in blue
        highlights: list of strings or a string
            list of sentences that will be visualized and colored in red           
        figsize: tuple
            size of the final image. The default is set to (80,200),
            which is 8000x20000 pixels and is appropriate for
            the stsb test dataset (2757 data points).

        An example of how the method can be used is provided 
        in sentence_similarity.py.
        Due to long loading times of the model, it is recommended 
        to use this method in the interactive mode: 
        python -i sentence_similarity.py
        """
        t1 = time.time()
        #Convert into list if it is a string
        if isinstance(highlights, str):
            highlights = [highlights]
        if len(highlights) == 1:
            highlights_embs = self.get_embeddings(highlights).reshape(1, -1)
        elif len(highlights) > 1:
            highlights_embs = self.get_embeddings(highlights)
        embs = self.get_embeddings(sentences)
        pca = PCA(n_components=2)
        X = pca.fit_transform(embs_together)
        plt.figure(figsize=(20,10))
        plt.scatter(X[:, 0], X[:, 1])
        for x, y in X:
            label = "{:.2f}".format(y)
            name = i
            label = f"{name}\n({x:.2f},{y:.2f})"

            plt.annotate(label, # this is the text
                        (x,y), # this is the point to label
                        textcoords="offset points", # how to position the text
                        xytext=(0,10), # distance from text to points (x,y)
                        ha='center') # horizontal alignment can be left, right or center
            i = i + 1
        plt.savefig(f'{randrange(0,10000)}plot.png')
        umap_model = umap.UMAP(n_neighbors=15, n_components=2, min_dist=0.5, spread=2, metric='cosine', random_state=42).fit(embs)
        umap_data_transformed = umap_model.transform(embs)

        plt.figure(figsize=(80,200))
        folder = "outputs"
        plot_name = f"plot-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.png"
        plot_path = os.path.join(folder,plot_name)
        legend_name = f'legend-{plot_name[:-4]}.txt'
        legend_path = os.path.join(folder,legend_name)
        with open(legend_path, 'w') as reader:
            for i, (x, y) in enumerate(umap_data_transformed):
                name = sentences[i]
                reader.write(f"{i}: {name} ({x:.2f},{y:.2f})\n")
                label = i
                plt.plot(x, y, 'bo')
                plt.text(x, y, label)
            if highlights:
                highlights_transformed = umap_model.transform(highlights_embs)
                for i, (x, y) in enumerate(highlights_transformed):
                    name = highlights[i]
                    label = f"{i}: {name}\n({x:.2f},{y:.2f})"
                    reader.write(f"Highlights: {i}: {name} ({x:.2f},{y:.2f})\n")
                    plt.plot(x, y, 'ro', markersize=30)
                    plt.text(x, y, label)
        plt.savefig(plot_path)
        t2 = time.time()
        print(f"Plot saved to {plot_path}. Time it took to generate: {t2-t1:.2f} seconds")
        print(f'{legend_path} stores the information about the data points on the plot.')

    def get_embeddings(self, sentences):
        return self.model.encode(sentences, convert_to_tensor=True)
+2760 −0

File added.

Preview size limit exceeded, changes collapsed.

+3.01 MiB
Loading image diff...
+5 −3
Changes for sentence_similarity.py: 5 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -3,9 +3,11 @@ from STSB_Dataset import STSB_Dataset

dataset = STSB_Dataset('data/stsbenchmark/sts-test.csv').as_dataframe

zero_model = SBERT_Model("Zero Model", 'models/nli-bert-large/', dataset)
#zero_model = SBERT_Model("Zero Model", 'models/nli-bert-large/', dataset)
ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset)

zero_model.print_statistics()
ft_model.print_statistics()
#zero_model.print_statistics()
#ft_model.print_statistics()

ft_model.visualize_embeddings(ft_model.sentences1+ft_model.sentences2, ["A cat is on the mat","A [MASK] is on the mat"])