Commit d05ee937 authored by podrazka's avatar podrazka
Browse files

Add methods 'get_word_importance' and 'get_html_word_importance' for...

Add methods 'get_word_importance' and 'get_html_word_importance' for generating heatmaps based on the difference in similarity caused by masking tokens
parent d752d144
Loading
Loading
Loading
Loading
+104 −4
Changes for SBERT_Model.py: 104 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -10,7 +10,8 @@ import numpy as np
import time
from datetime import datetime
import os

import copy
import spacy

class SBERT_Model:
    def __init__(self, name, filepath, dataset, probing=False):
@@ -26,7 +27,7 @@ class SBERT_Model:
        self.labels = dataset['label'].tolist()
        self.embeddings1 = self.get_embeddings(self.sentences1)
        self.embeddings2 = self.get_embeddings(self.sentences2)
        self.cosine_scores = self.get_cosine_scores()
        self.cosine_scores = self.get_cosine_scores(self.embeddings1, self.embeddings2)
        self.preds = self.get_preds()

    def sentence_clusters(self):
@@ -113,6 +114,105 @@ class SBERT_Model:
        print(f"Plot saved to {plot_path}. Time it took to generate: {t2-t1:.2f} seconds")
        print(f'{legend_path} stores the information about the data points on the plot.')

    def get_word_importance(self,sentence_pair, mask="[MASK]"):
        nlp = spacy.load("en_core_web_sm")
        sentence1 = nlp(sentence_pair[0])
        sentence2 = nlp(sentence_pair[1])
        unmasked_embs = self.get_embeddings([sentence_pair[0], sentence_pair[1]])
        unmasked_similarity = self.get_cosine_scores(unmasked_embs[0], unmasked_embs[1])
        masked_pairs = []
        for i in range(len(sentence1)):
            masked_sentence = []
            for y in range(len(sentence1)):
                if y == i:
                    masked_sentence.append(mask)
                    masked_word = sentence1[y].text
                else:
                    masked_sentence.append(sentence1[y].text)
            masked_sentence = " ".join(masked_sentence)
            masked_pair = {"masked word": masked_word, "sentence1": masked_sentence, "sentence2": sentence_pair[1]}
            masked_pairs.append(masked_pair)
        for i in range(len(sentence2)):
            masked_sentence = []
            for y in range(len(sentence2)):
                if y == i:
                    masked_sentence.append(mask)
                    masked_word = sentence2[y].text
                else:
                    masked_sentence.append(sentence2[y].text)
            masked_sentence = " ".join(masked_sentence)
            masked_pair = {"masked word": masked_word, "sentence1": sentence_pair[0], "sentence2": masked_sentence}
            masked_pairs.append(masked_pair)
        scores = []
        for masked_pair in masked_pairs:
            embs = self.get_embeddings([masked_pair["sentence1"], masked_pair["sentence2"]])
            scores.append((masked_pair, float(abs(unmasked_similarity - self.get_cosine_scores(embs[0], embs[1])))))
#TODO: Find out more about the words that cause high deviation from unmasked similarity.
#         sorted_scores = copy.deepcopy(scores)
#         sorted_scores.sort(key = lambda x: x[1], reverse=True)
#         print(f"Pair: {sentence_pair}, Gold: {gold}")
#         for score in sorted_scores:
#             masked_pair = score[0]
#             print(f'1. {masked_pair["sentence1"]}\n \
# 2.{masked_pair["sentence2"]}\n \
# Deviation: {score[1]:.3f}, Masked word: {masked_pair["masked word"]}\n')
        html_output = []
        html_output.append('<div class="sentence">\n')
        for i, score in enumerate(scores):
            masked_pair = score[0]
            html_output.append(f'<div style="background-color: rgba(0, 160, 252, {score[1]:.2f});">{masked_pair["masked word"]}<br>{score[1]:.2f}</div>\n')
            if i == len(sentence1)-1:
                html_output.append('</div>\n<div class="sentence">\n')
        html_output.append('</div>\n')
        return html_output

    def get_html_word_importance(self, sentence_pairs, mask="[MASK]"):
        top = """<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <title>SBERT Word Importance</title>
  <meta name="description" content="Visualizing SBERT sentence pairs importance of words">
<style>
.sentence{
    display:flex;
}
.sentence > div{
    border: 1px solid gray;
    
    min-width: 100px;
    min-height: 20px;
    padding: 20px;
    align-items: center;
    text-align: center;
}

</style>
</head>

<body>
    <h1>SBERT Word Importance Visualization</h1>
"""

        bottom = "</body>\n</html>"

        folder = "outputs"
        html_name = f"heatmap-{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.html"
        html_path = os.path.join(folder,html_name)

        with open(html_path, 'w') as reader:
            reader.write(top)
            reader.write(f'<h2>{self.name}</h2>\n')
            reader.write(f'<h3>Token used for masking:{mask}</h3>\n')
            for i, sentence_pair in enumerate(sentence_pairs):
                print(i)
                reader.write(f'<div id="{i}" class="sentence_pair">{i+1}.\n')
                sentence_divs = self.get_word_importance(sentence_pair, mask=mask)
                for sentence_div in sentence_divs:
                    reader.write(sentence_div)
                reader.write('</div>\n')
            reader.write(bottom)
            
    def get_embeddings(self, sentences):
        return self.model.encode(sentences, convert_to_tensor=True)

@@ -126,8 +226,8 @@ class SBERT_Model:
            preds.append(self.cosine_scores[i][i])
        return preds

    def get_cosine_scores(self):
        return util.pytorch_cos_sim(self.embeddings1, self.embeddings2)
    def get_cosine_scores(self, emb1, emb2):
        return util.pytorch_cos_sim(emb1, emb2)

    def get_pearson(self):
        return pearsonr(self.labels, self.preds)[0]
+2109 −0

File added.

Preview size limit exceeded, changes collapsed.

+7 −1
Changes for sentence_similarity.py: 7 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -9,5 +9,11 @@ ft_model = SBERT_Model("Fine-tuned Model", 'models/stsb-bert-large/', dataset)
#zero_model.print_statistics()
#ft_model.print_statistics()

ft_model.visualize_embeddings(ft_model.sentences1+ft_model.sentences2, ["A cat is on the mat","A [MASK] is on the mat"])
#ft_model.visualize_embeddings(ft_model.sentences1+ft_model.sentences2, ["A cat is on the mat","A [MASK] is on the mat"])

#Make a list with sentence pairs for the heatmap
sentence_pairs = []
for i in range(2):
    sentence_pairs.append((ft_model.sentences1[i],ft_model.sentences2[i]))
#generate the heatmap with default mask == "[MASK]"
ft_model.get_html_word_importance(sentence_pairs)
 No newline at end of file