BertForSequenceClassification.from_pretrained about bertviz HOT 17 CLOSED

from bertviz import head_view
from transformers import BertTokenizer, BertForSequenceClassification

%%javascript
require.config({
  paths: {
      d3: '//cdnjs.cloudflare.com/ajax/libs/d3/3.4.8/d3.min',
      jquery: '//ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min',
  }
});

model_path = '/bert/outputs/'
save_model = BertForSequenceClassification.from_pretrained('/bert/outputs/pytorch_model.bin', num_labels=2)
torch.save(save_model.state_dict(), model_path)

from bertviz.transformers_attn import BertModel, BertTokenizer
from bertviz.head_view_bert import show
from bertviz.transformers_attn import BertForSequenceClassification

model_version = 'bert-base-cased'
do_lower_case = True
model_state_dict = torch.load(model_path)
assert model_state_dict is not None

model = BertModel.from_pretrained(model_version, state_dict=model_state_dict,output_attentions=True,num_labels=2)
tokenizer = BertTokenizer.from_pretrained(model_version, do_lower_case=do_lower_case)

sentence_a = "The cat sat on the mat"
sentence_b = "The cat lay on the rug"

inputs = tokenizer.encode_plus(sentence_a, sentence_b, return_tensors='pt', add_special_tokens=True)
token_type_ids = inputs['token_type_ids']
input_ids = inputs['input_ids']
attention = model(input_ids, token_type_ids=token_type_ids)[-1]
input_id_list = input_ids[0].tolist() # Batch index 0
tokens = tokenizer.convert_ids_to_tokens(input_id_list)
head_view(attention, tokens)

I1125 08:20:57.783921 140693827761984 configuration_utils.py:148] loading configuration file /bert//outputs/paraphrase/pytorch_model.bin
---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-5-b4e6db36dc04> in <module>
      1 model_path = '/home/rpc/DeepL/Paraphrase/outputs/paraphrase/'
----> 2 save_model = BertForSequenceClassification.from_pretrained('/outputs/pytorch_model.bin', num_labels=2)
      3 torch.save(save_model.state_dict(), model_path)
      4 
      5 from bertviz.transformers_attn import BertModel, BertTokenizer

~/.local/lib/python3.6/site-packages/transformers/modeling_utils.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    285                 cache_dir=cache_dir, return_unused_kwargs=True,
    286                 force_download=force_download,
--> 287                 **kwargs
    288             )
    289         else:

~/.local/lib/python3.6/site-packages/transformers/configuration_utils.py in from_pretrained(cls, pretrained_model_name_or_path, **kwargs)
    152 
    153         # Load config
--> 154         config = cls.from_json_file(resolved_config_file)
    155 
    156         if hasattr(config, 'pruned_heads'):

~/.local/lib/python3.6/site-packages/transformers/configuration_utils.py in from_json_file(cls, json_file)
    184         """Constructs a `BertConfig` from a json file of parameters."""
    185         with open(json_file, "r", encoding='utf-8') as reader:
--> 186             text = reader.read()
    187         return cls.from_dict(json.loads(text))
    188 

/usr/lib/python3.6/codecs.py in decode(self, input, final)
    319         # decode input (taking the buffer into account)
    320         data = self.buffer + input
--> 321         (result, consumed) = self._buffer_decode(data, self.errors, final)
    322         # keep undecoded input until the next call
    323         self.buffer = data[consumed:]

UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte

from bertviz.pytorch_transformers_attn import BertForSequenceClassification
from bertviz.pytorch_transformers_attn import BertModel, BertTokenizer
from bertviz.head_view import show

%%javascript
require.config({
  paths: {
      d3: '//cdnjs.cloudflare.com/ajax/libs/d3/3.4.8/d3.min',
      jquery: '//ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min',
  }
});

import torch
from pytorch_pretrained_bert import BertModel, BertTokenizer, modeling
from pytorch_pretrained_bert.file_utils import PYTORCH_PRETRAINED_BERT_CACHE
from pathlib import Path

save_dir = Path('/home/bert/savedmodelfolder/')
save_dir.mkdir(exist_ok=True)
bert_model = 'bert-base-cased'
save_file = save_dir / modeling.WEIGHTS_NAME
config_file = save_dir / modeling.CONFIG_NAME
model_base = BertModel.from_pretrained(
    bert_model,
    cache_dir=PYTORCH_PRETRAINED_BERT_CACHE / 'distributed_{}'.format(-1)
)


model_type = 'bert'
model_version = 'bert-base-cased'
do_lower_case = False


# Loading
model_state_dict = torch.load(save_file)
state_dict_with_prefix = {}
for key, value in model_state_dict.items():
    state_dict_with_prefix['bert.' + key] = value
model_loaded = BertModel.from_pretrained(save_dir, state_dict=state_dict_with_prefix)

tokenizer = BertTokenizer.from_pretrained(model_version, do_lower_case=do_lower_case)
sentence_a = "The cat sat on the mat"
sentence_b = "The cat lay on the rug"
show(model_loaded, model_type, tokenizer, sentence_a, sentence_b)

AttributeError                            Traceback (most recent call last)
<ipython-input-15-a0212be47ce3> in <module>
     30 sentence_a = "The cat sat on the mat"
     31 sentence_b = "The cat lay on the rug"
---> 32 show(model_loaded, model_type, tokenizer, sentence_a, sentence_b)

~/bertviz/head_view.py in show(model, model_type, tokenizer, sentence_a, sentence_b)
     57         os.path.join(os.getcwd(), os.path.dirname(__file__)))
     58     vis_js = open(os.path.join(__location__, 'head_view.js')).read()
---> 59     attn_data = get_attention(model, model_type, tokenizer, sentence_a, sentence_b)
     60     params = {
     61         'attention': attn_data,

~/bertviz/attention.py in get_attention(model, model_type, tokenizer, sentence_a, sentence_b, include_queries_and_keys)
     57     else:
     58         if model_type == 'bert':
---> 59             tokens_a = [tokenizer.cls_token] + tokenizer.tokenize(sentence_a) + [tokenizer.sep_token]
     60             tokens_b = tokenizer.tokenize(sentence_b) + [tokenizer.sep_token]
     61             token_type_ids = torch.LongTensor([[0] * len(tokens_a) + [1] * len(tokens_b)])

AttributeError: 'BertTokenizer' object has no attribute 'cls_token'