Loading attention_visualization.py 0 → 100644 +40 −0 Changes for attention_visualization.py: 40 added lines, 0 removed lines. Original line number Diff line number Diff line import torch from transformers import BertConfig, BertTokenizer, BertModel import seaborn as sns import matplotlib.pyplot as plt model_type = 'sts-bert-large' config = BertConfig.from_pretrained(model_type) config.output_attentions = True model = BertModel.from_pretrained(model_type, config=config).to('cpu') tokenizer = BertTokenizer.from_pretrained(model_type) text = 'A dog standing in the water' tok = tokenizer.tokenize(text) pos = 2 ids = torch.tensor(tokenizer.convert_tokens_to_ids(tok)).unsqueeze(0).to('cpu') with torch.no_grad(): output = model(ids) attentions = torch.cat(output[2]).to('cpu') attentions = attentions.permute(2, 1, 0, 3) layers = len(attentions[0][0]) heads = len(attentions[0]) seqlen = len(attentions) attentions_pos = attentions[pos] cols = 2 rows = int(heads / cols) fig, axes = plt.subplots(rows, cols, figsize=(14, 30)) axes = axes.flat print(f'Attention weights for token {tok[pos]}') for i, att in enumerate(attentions_pos): sns.heatmap(att, vmin=0, vmax=1, ax=axes[i], xticklabels=tok) axes[i].set_title(f'head {i}') axes[i].set_ylabel('layers') avg_attention = attentions_pos.mean(dim=0) sns.heatmap(avg_attention, vmin=0, vmax=1, xticklabels=tok) Loading
attention_visualization.py 0 → 100644 +40 −0 Changes for attention_visualization.py: 40 added lines, 0 removed lines. Original line number Diff line number Diff line import torch from transformers import BertConfig, BertTokenizer, BertModel import seaborn as sns import matplotlib.pyplot as plt model_type = 'sts-bert-large' config = BertConfig.from_pretrained(model_type) config.output_attentions = True model = BertModel.from_pretrained(model_type, config=config).to('cpu') tokenizer = BertTokenizer.from_pretrained(model_type) text = 'A dog standing in the water' tok = tokenizer.tokenize(text) pos = 2 ids = torch.tensor(tokenizer.convert_tokens_to_ids(tok)).unsqueeze(0).to('cpu') with torch.no_grad(): output = model(ids) attentions = torch.cat(output[2]).to('cpu') attentions = attentions.permute(2, 1, 0, 3) layers = len(attentions[0][0]) heads = len(attentions[0]) seqlen = len(attentions) attentions_pos = attentions[pos] cols = 2 rows = int(heads / cols) fig, axes = plt.subplots(rows, cols, figsize=(14, 30)) axes = axes.flat print(f'Attention weights for token {tok[pos]}') for i, att in enumerate(attentions_pos): sns.heatmap(att, vmin=0, vmax=1, ax=axes[i], xticklabels=tok) axes[i].set_title(f'head {i}') axes[i].set_ylabel('layers') avg_attention = attentions_pos.mean(dim=0) sns.heatmap(avg_attention, vmin=0, vmax=1, xticklabels=tok)