Commit a4161080 authored by Aileen Reichelt's avatar Aileen Reichelt
Browse files

Add mostly unchanged base for attention visualization code

parent dc8fd408
Loading
Loading
Loading
Loading
+40 −0
Changes for attention_visualization.py: 40 added lines, 0 removed lines.
Original line number Diff line number Diff line
import torch
from transformers import BertConfig, BertTokenizer, BertModel
import seaborn as sns
import matplotlib.pyplot as plt

model_type = 'sts-bert-large'
config = BertConfig.from_pretrained(model_type)
config.output_attentions = True
model = BertModel.from_pretrained(model_type, config=config).to('cpu')
tokenizer = BertTokenizer.from_pretrained(model_type)

text = 'A dog standing in the water'
tok = tokenizer.tokenize(text)
pos = 2

ids = torch.tensor(tokenizer.convert_tokens_to_ids(tok)).unsqueeze(0).to('cpu')
with torch.no_grad():
    output = model(ids)
attentions = torch.cat(output[2]).to('cpu')
attentions = attentions.permute(2, 1, 0, 3)
layers = len(attentions[0][0])
heads = len(attentions[0])
seqlen = len(attentions)

attentions_pos = attentions[pos]

cols = 2
rows = int(heads / cols)

fig, axes = plt.subplots(rows, cols, figsize=(14, 30))
axes = axes.flat
print(f'Attention weights for token {tok[pos]}')

for i, att in enumerate(attentions_pos):
    sns.heatmap(att, vmin=0, vmax=1, ax=axes[i], xticklabels=tok)
    axes[i].set_title(f'head {i}')
    axes[i].set_ylabel('layers')

avg_attention = attentions_pos.mean(dim=0)
sns.heatmap(avg_attention, vmin=0, vmax=1, xticklabels=tok)