Commit 795071ba authored by Aileen Reichelt's avatar Aileen Reichelt
Browse files

Did not use LIT

parent 53789f17
Loading
Loading
Loading
Loading

runLIT.py

deleted100755 → 0
+0 −437
Changes for runLIT.py: 0 added lines, 437 removed lines.
Original line number Diff line number Diff line
"""
This script needs a LIT-compatible environment with transformers==2.11.0 and Python==3.7 (see conda environment file lit.yml)

This script wraps the model(s) and data from runBertForMultiChoiceMaskedLM.py according to LIT standards.
By running this script you can inspect the models in LIT.
Beware of the RAM requirements when loading all models. By default, we load 2 of 3 models. You can change which models are loaded by (un)commenting any of these lines in the models dict at the end of the script:
models = {
      'mcbert_frozen_encoder': LIT_MC_BERT('models/mcbert_frozen_encoder', restrict_vocab=True),
      'mcbert_fully_finetuned': LIT_MC_BERT('models/mcbert_fully_finetuned', restrict_vocab=True),
      #'mcbert_base': LIT_MC_BERT(BASE_MODEL, restrict_vocab=False),
    }
To load multiple models at once you need >8GB RAM. Loading multiple models enables you to compare them directly from within the LIT tool.
"""

import lit_nlp
from lit_nlp import dev_server
from lit_nlp import server_flags
from lit_nlp.api import model as lit_model
from lit_nlp.api import dataset as lit_dataset
from lit_nlp.api import types as lit_types
from lit_nlp.lib import utils
import transformers
import tensorflow as tf
import ast
import torch
import argparse
from tqdm import tqdm
from transformers import BertTokenizer, BertForMaskedLM
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
import numpy as np
import re
from typing import Dict, List, Tuple

device = "cuda:0" if torch.cuda.is_available() else "cpu"

# underlying pretrained LM
BASE_MODEL = 'bert-large-uncased-whole-word-masking'

# choices for the multi-choice setup; sorted alphabetically
CHOICES = ["always", "never", "often", "rarely", "sometimes"]

# back-mapping dictionary of strings to integer labels
CHOICES2ID = {k: v for v, k in enumerate(CHOICES)}

# retrieved IDs from vocab; 2467 is always, 2196 is never, and so on
IDS = [2467, 2196, 2411, 6524, 2823]

ENUMIDS = dict(enumerate(IDS))


class LIT_AlwaysNeverDataset(lit_dataset.Dataset):
    
    # this is a LIT-specific wrapper for our AlwaysNeverDataset
    
    MC_labels = ['always', 'never', 'often', 'rarely', 'sometimes']

    def __init__(self, path):

        with open('data/challenge_coffee_cats_quantifiers_coffee_cats_quantifiers_dev.jsonl',
                  'r', encoding='utf-8') as d:
            dataset = d.read().splitlines()

        self.data = []
        self.labels = []

        for line in dataset:
            # we exploit that the data is already in a dictionary structure and convert the string representation of the dict to an actual dict
            line_as_dict = ast.literal_eval(line)
            
            # value of 'question' is a dictionary with the keys 'choices' and 'stem' (the latter being the sentence with the masked token)
            question_dict = line_as_dict['question']
            # use pretrained tokenizer to prepare input ids
            self.data.append(question_dict['stem'])
            
            # how the value for 'choices' looks like:
            # [{"label": "A", "text": "rarely"}, {"label": "B", "text": "sometimes"}, {"label": "C", "text": "never"}, {"label": "D", "text": "often"}, {"label": "E", "text": "always"}]
            # the order of the labels is the same for each data sample but the text changes!
            # sort the above list of dictionaries by 'text' key (alphabetically)
            sorted_choices = sorted(question_dict['choices'], key=lambda k: k['text'])
            
            # gold label is given in line_as_dict['answerKey']
            label = [i for i,choice in enumerate(sorted_choices) if choice['label']==line_as_dict['answerKey']]
            assert(len(label)==1)
            self.labels.append(CHOICES[label[0]])
            
        # Store as a list of dicts, conforming to self.spec()
        self._examples = [{
          'sentence': sentence,
          'label': label,
        } for sentence, label in zip(self.data, self.labels)]

    def spec(self):
        return {
          'sentence': lit_types.TextSegment(),
          'label': lit_types.CategoryLabel(vocab=self.MC_labels),
        }

class LIT_STSB_Dataset(lit_dataset.Dataset):
    
    # this is a LIT-specific wrapper for our STS-B Dataset.
    


    def __init__(self, path):

        with open(path,
                  'r', encoding='utf-8') as d:
            dataset = d.read().splitlines()

        self.data = []
        self.labels = []


        for line in dataset:
            datapoint = line.split('\t')
            score = float(datapoint[4])
            score = score/5

            st1 = datapoint[5]
            st2 = datapoint[6]
            self.data.append([st1,st2])
            self.labels.append(score)

            
        # Store as a list of dicts, conforming to self.spec()
        self._examples = [{
          'sentence1': dp[0],
          'sentence2': dp[1],
          'label': label,
        } for dp, label in zip(self.data, self.labels)]

    def spec(self):
        return {
          'sentence1': lit_types.TextSegment(),
          'sentence2': lit_types.TextSegment(),
          'label': lit_types.RegressionScore(),
        }

class BertForMultiChoiceMaskedLM(BertForMaskedLM):
    
    # this is our custom model from runBertForMultiChoiceMaskedLM.py
    
    def __init__(self, config):
        super().__init__(config)
        
        self.loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-1)
        self.softmax = torch.nn.Softmax(dim=0)

    def set_tokenizer(self, tokenizer):
        self.tokenizer = tokenizer

    def freeze(self):
        for param in self.base_model.parameters():
            param.requires_grad = False

    def forward(self, input_ids, attention_mask=None, labels=None, verbose=False,
                restrict_vocab=False, input_embs=None):
        
        if type(input_embs) == type(None):
            last_hidden_states, pooler_output, hidden_states, attentions = \
                self.bert(input_ids, attention_mask=attention_mask)
        else:
            last_hidden_states, pooler_output, hidden_states, attentions = \
                self.bert(inputs_embeds=input_embs)


        logits = self.cls(last_hidden_states)
        
        loss = 0
        predictions = list()
        choice_logit_lst = list()
        
        # iterate over elements in batch
        for i, element_input_ids in enumerate(input_ids):
            # [MASK] token is 103 in vocab, search for position in input with element_input_ids.index(103)
            # [2467, 2196, 2411, 6524, 2823] is the list of indices of the candidate tokens in the 
            # vocab (see commentary at the beginning)
            choice_logits = logits[i, element_input_ids.tolist().index(103), IDS]
            choice_logit_lst.append(choice_logits)
            # torch.max(tensor, dim) returns (values, indices)
            predictions.append(torch.max(choice_logits, 0)[1].item())
            loss += self.loss_fct(choice_logits.unsqueeze(0), labels[i].unsqueeze(0))
            if verbose:
                choice_probs = self.softmax(choice_logits)
                print(self.tokenizer.decode(element_input_ids))
                print(["%s: %2.1f%%" %(choice, 100 * score) for choice, score in zip(CHOICES, choice_probs)],
                      "True label:", labels[i])
        
        if restrict_vocab:
            logits = logits[:, :, IDS]
        choice_logits = torch.stack(choice_logit_lst)
        if len(choice_logits.shape) == 1:
            choice_logits = choice_logits.unsqueeze(0)
        return (loss, predictions, logits, choice_logits, last_hidden_states, 
                hidden_states, attentions)
    


class LIT_MC_BERT(lit_model.Model):
    
    # this is a LIT-specific wrapper for our custom BertForMultiChoiceMaskedLM model

    MC_labels = ['always', 'never', 'often', 'rarely', 'sometimes']
    MASK_TOKEN = "[MASK]"

    @property
    def max_seq_length(self):
        return self.model.config.max_position_embeddings

    def __init__(self, model_path, batch_size=8, restrict_vocab=False):
        super().__init__()
        self.tokenizer = BertTokenizer.from_pretrained(BASE_MODEL)
        model_config = transformers.AutoConfig.from_pretrained(
            model_path,
            output_hidden_states=True,
            output_attentions=True,
        )
        self.model = BertForMultiChoiceMaskedLM.from_pretrained(model_path,
                                                                config=model_config)
        self.model = self.model.to(device)
        self.model.set_tokenizer(self.tokenizer)
        self.restrict_vocab = restrict_vocab

        if self.restrict_vocab:
            self.top_k = 5
        else:
            self.top_k = 10

        self.batch_size = batch_size
        
    # LIT API implementation
    def max_minibatch_size(self):
        return self.batch_size

    def _get_topk_tokens(self, scores: np.ndarray, enum=None) -> List[List[Tuple[str, float]]]:
        """Convert raw scores to top-k token predictions."""
        # scores is [num_tokens, vocab_size]
        # Find the vocab indices of top k predictions, at each token.
        # np.argpartition is faster than a full argsort for k << V,
        # but we need to sort the output after slicing (see below).
        index_array = np.argpartition(scores, -self.top_k, axis=1)[:, -self.top_k:]
        # These are each [num_tokens, tok_k]
        if enum:
            filter_array = np.array([[enum[y] for y in x] for x in index_array])
            top_tokens = [self.tokenizer.convert_ids_to_tokens(idxs) for idxs in filter_array]
        else:
            top_tokens = [self.tokenizer.convert_ids_to_tokens(idxs) for idxs in index_array]
        top_scores = np.take_along_axis(scores, index_array, axis=1)
        # Convert to a list of lists of (token, score) pairs,
        # where inner lists are sorted in descending order of score.
        return [
          sorted(list(zip(toks, scores)), key=lambda ab: -ab[1])
          for toks, scores in zip(top_tokens, top_scores)
        ]
    
    def _preprocess(self, inputs):
        tokenized_texts = [self.tokenizer.tokenize(ex['sentence']) for ex in inputs]
        encoded_input = self.tokenizer.batch_encode_plus(
            tokenized_texts,
            is_pretokenized=True,
            return_tensors="pt",
            # return_tensors="tf",
            add_special_tokens=True,
            max_length=self.max_seq_length,
            pad_to_max_length=True)
        max_tokens = tf.reduce_max(
            tf.reduce_sum(encoded_input["attention_mask"], axis=1))
        encoded_input = {k: v[:, :max_tokens] for k, v in encoded_input.items()}
        return encoded_input

    
    def _postprocess(self, output: Dict[str, np.ndarray]):
        """Postprocess, modifying output dict in-place."""
        # Slice to remove padding, omitting initial [CLS] and final [SEP]
        ntok = output.pop("ntok")
        slicer = slice(1, ntok - 1)
        output["tokens"] = self.tokenizer.convert_ids_to_tokens(output.pop("input_ids")[slicer])
        probas = output.pop("probas")

        # Predictions at every position, regardless of masking.
        if self.restrict_vocab:
            output["pred_tokens"] = self._get_topk_tokens(probas[slicer], enum=ENUMIDS)
        else:
            output["pred_tokens"] = self._get_topk_tokens(probas[slicer], enum=None)
        # Trim down to only the mask positions, to avoid sending a huge amount of data.
        for i, token in enumerate(output["tokens"]):
            if token != self.MASK_TOKEN:
                output["pred_tokens"][i] = []
        
        output["input_embs"] = output["input_embs"][slicer]
        output["token_grad"] = output["input_emb_grad"][slicer]
        # output["grad_class"] = self.config.labels[output["grad_class"]]
        for key in output:
            if not re.match(r"layer_(\d+)/attention", key):
                continue
            # Select only real tokens, since most of this matrix is padding.
            output[key] = output[key][:, :ntok, :ntok].transpose((0, 2, 1))
            # Make a copy of this array to avoid memory leaks, since NumPy otherwise
            # keeps a pointer around that prevents the source array from being GCed.
            output[key] = output[key].copy()  # pytype: disable=attribute-error
        return output
    
    
    # LIT API implementations
    def predict_minibatch(self, inputs, config=None):
        # print inputs if you want to track speed of processing in console
        # print(inputs)
        self.model.zero_grad()
        """Predict on a single minibatch of examples."""
        # If input has a 'tokens' field, use that. Otherwise tokenize the text.
        # Process to ids, add special tokens, and compute segment ids and masks.
        encoded_input = self._preprocess(inputs)
        word_embeddings = self.model.bert.embeddings.word_embeddings.to(device)
        input_ids = encoded_input["input_ids"].to(device)
        input_embs = word_embeddings(input_ids.to(device))
        input_embs.retain_grad()
        if type(inputs[0]["label"]) is str:
            labels = torch.LongTensor([torch.tensor(CHOICES2ID[x["label"]]).long() for x in inputs])
        else:
            labels = torch.LongTensor([torch.tensor(x["label"]).long() for x in inputs])
        loss, predictions, logits, choice_logits, last_hidden_states, hidden_states, attentions  = \
            self.model(input_ids = encoded_input["input_ids"].to(device), 
                       attention_mask = encoded_input["attention_mask"].to(device), labels=labels.to(device),
                       restrict_vocab=self.restrict_vocab,
                       input_embs=input_embs.to(device))
        
        batched_outputs = {
            "input_ids": encoded_input["input_ids"],
            "ntok": torch.sum(encoded_input["attention_mask"], dim=1),
            #"ntok": tf.reduce_sum(encoded_input["attention_mask"], axis=1),
            "cls_emb": last_hidden_states[:, 0],  # last layer, first token
            "input_embs": input_embs
        }
        
        assert len(attentions) == self.model.config.num_hidden_layers
        for i, layer_attention in enumerate(attentions):
            batched_outputs[f"layer_{i}/attention"] = layer_attention
        
        batched_outputs["choice_probas"] = torch.nn.functional.softmax(choice_logits, dim=-1)
        batched_outputs["probas"] = torch.nn.functional.softmax(logits, dim=-1)
        # batched_outputs["probas"] = tf.nn.softmax(logits, axis=-1)

        arg_max = torch.argmax(batched_outputs["choice_probas"].cpu(), dim=-1, keepdim=True).numpy()
        # arg_max = tf.math.argmax(batched_outputs["probas"].numpy(), axis=-1).numpy()

        grad_classes = [ex.get("grad_class", arg_max[i]) for (i, ex) in
                        enumerate(inputs)]

        # Convert the class names to indices if needed.
        grad_classes = [CHOICES2ID(label) if isinstance(label, str) else label
                        for label in grad_classes]
        # grad_classes = [self.config.labels.index(label)
                        # if isinstance(label, str) else label
                        # for label in grad_classes]

        gather_indices = torch.LongTensor(grad_classes).to(device)
        # gather_indices = torch.LongTensor(list(enumerate(grad_classes)))

        scalar_pred_for_gradients = torch.gather(batched_outputs["choice_probas"], -1,
                                                 gather_indices).squeeze()
        # scalar_pred_for_gradients = tf.gather_nd(batched_outputs["probas"],
                                                 # gather_indices)
        
        for scalar in scalar_pred_for_gradients:
            scalar.backward(retain_graph=True)
        batched_outputs["input_emb_grad"] = input_embs.grad
        # batched_outputs["input_emb_grad"] = tape.gradient(
            # scalar_pred_for_gradients, input_embs)

        batched_outputs["grad_class"] = torch.FloatTensor(grad_classes)
        # batched_outputs["grad_class"] = tf.convert_to_tensor(grad_classes)

        # List of dicts, one per example.
        detached_outputs = {k: v.cpu().detach().numpy() for k, v in batched_outputs.items()}
        unbatched_outputs = utils.unbatch_preds(detached_outputs)

        # Postprocess to remove padding and decode predictions.
        return map(self._postprocess, unbatched_outputs)


    def input_spec(self):
        return {
            "sentence": lit_types.TextSegment(),
            "tokens": lit_types.Tokens(required=False),
            "label": lit_types.CategoryLabel(required=False, vocab=CHOICES),
            "input_embs": lit_types.TokenEmbeddings(align="tokens", required=False),
            "grad_class": lit_types.CategoryLabel(required=False, vocab=CHOICES)
        }
  
    def output_spec(self):
        ret = {
            "tokens": lit_types.Tokens(parent="text"),
            "pred_tokens": lit_types.TokenTopKPreds(align="tokens"),
            "cls_emb": lit_types.Embeddings(),
            "choice_probas": lit_types.MulticlassPreds(parent="label", vocab=CHOICES, null_idx=0),
            "input_embs": lit_types.TokenEmbeddings( align="tokens"),
            "grad_class": lit_types.CategoryLabel(required=False, vocab=CHOICES),
            "token_grad": lit_types.TokenGradients(align="tokens", grad_for="input_embs",
                                                   grad_target="grad_class")
        }
        for i in range(self.model.config.num_hidden_layers):
            ret[f"layer_{i}/attention"] = lit_types.AttentionHeads(
                align=("tokens", "tokens"))
        return ret


def main():
    datasets = {
      'always_never_test': LIT_AlwaysNeverDataset('data/challenge_coffee_cats_quantifiers_coffee_cats_quantifiers_dev.jsonl'),
      'sts-b': LIT_STSB_Dataset('data/stsbenchmark/sts-test.csv')
    }
    
    print("add or delete comments in code to select models")
    # comment out models in case of memory issues; >8GB RAM is needed for multiple models
    models = {
        #'mcbert_frozen_encoder': LIT_MC_BERT('/mnt/proj/wiesenbach/mcbert/models/mcbert_frozen_encoder', restrict_vocab=True,
                                             #batch_size=8),
        'stsb-bert-large': LIT_MC_BERT('models/stsb-bert-large/0_BERT', restrict_vocab=False),
        'nli-bert-large': LIT_MC_BERT('models/nli-bert-large/0_BERT', restrict_vocab=False),
        'mcbert_fully_finetuned': LIT_MC_BERT('/mnt/proj/wiesenbach/mcbert/models/mcbert_fully_finetuned', restrict_vocab=True,
                                              batch_size=8),
        #'mcbert_base': LIT_MC_BERT(BASE_MODEL, restrict_vocab=False,
                                   #batch_size=8),
    }
    

    flags = server_flags.get_flags()
    flags["port"] = 5420
    lit_demo = lit_nlp.dev_server.Server(models, datasets, **flags)
    lit_demo.serve()


if __name__ == '__main__':
    main()