Initializing VideoChat
Loading VIT. Use fp16: fp32
Temporal downsample: False
No L_MHRA: True
Double L_MHRA: False
GMHRA index: [38, 37, 36, 35, 34, 33, 32, 31]
GMHRA dropout: 0.5
Drop path rate: 0.0
Drop path rate: 0.0
Drop path rate: 0.0
Drop path rate: 0.0
Drop path rate: 0.0
Drop path rate: 0.0
Drop path rate: 0.0
Drop path rate: 0.0
Load ViT model from: D:/GIT_Project/Ask-Anything/Bilp2/eva_vit_g.pth
Inflate: patch_embed.proj.weight, torch.Size([1408, 3, 14, 14]) => torch.Size([1408, 3, 1, 14, 14])
Init center: True
_IncompatibleKeys(missing_keys=['gmhra_cls_token', 'gmhra.0.dpe.weight', 'gmhra.0.dpe.bias', 'gmhra.0.attn.in_proj_weight', 'gmhra.0.attn.in_proj_bias', 'gmhra.0.attn.out_proj.weight', 'gmhra.0.attn.out_proj.bias', 'gmhra.0.ln_1.weight', 'gmhra.0.ln_1.bias', 'gmhra.0.mlp.c_fc.weight', 'gmhra.0.mlp.c_fc.bias', 'gmhra.0.mlp.c_proj.weight', 'gmhra.0.mlp.c_proj.bias', 'gmhra.0.ln_2.weight', 'gmhra.0.ln_2.bias', 'gmhra.0.ln_3.weight', 'gmhra.0.ln_3.bias', 'gmhra.1.dpe.weight', 'gmhra.1.dpe.bias', 'gmhra.1.attn.in_proj_weight', 'gmhra.1.attn.in_proj_bias', 'gmhra.1.attn.out_proj.weight', 'gmhra.1.attn.out_proj.bias', 'gmhra.1.ln_1.weight', 'gmhra.1.ln_1.bias', 'gmhra.1.mlp.c_fc.weight', 'gmhra.1.mlp.c_fc.bias', 'gmhra.1.mlp.c_proj.weight', 'gmhra.1.mlp.c_proj.bias', 'gmhra.1.ln_2.weight', 'gmhra.1.ln_2.bias', 'gmhra.1.ln_3.weight', 'gmhra.1.ln_3.bias', 'gmhra.2.dpe.weight', 'gmhra.2.dpe.bias', 'gmhra.2.attn.in_proj_weight', 'gmhra.2.attn.in_proj_bias', 'gmhra.2.attn.out_proj.weight', 'gmhra.2.attn.out_proj.bias', 'gmhra.2.ln_1.weight', 'gmhra.2.ln_1.bias', 'gmhra.2.mlp.c_fc.weight', 'gmhra.2.mlp.c_fc.bias', 'gmhra.2.mlp.c_proj.weight', 'gmhra.2.mlp.c_proj.bias', 'gmhra.2.ln_2.weight', 'gmhra.2.ln_2.bias', 'gmhra.2.ln_3.weight', 'gmhra.2.ln_3.bias', 'gmhra.3.dpe.weight', 'gmhra.3.dpe.bias', 'gmhra.3.attn.in_proj_weight', 'gmhra.3.attn.in_proj_bias', 'gmhra.3.attn.out_proj.weight', 'gmhra.3.attn.out_proj.bias', 'gmhra.3.ln_1.weight', 'gmhra.3.ln_1.bias', 'gmhra.3.mlp.c_fc.weight', 'gmhra.3.mlp.c_fc.bias', 'gmhra.3.mlp.c_proj.weight', 'gmhra.3.mlp.c_proj.bias', 'gmhra.3.ln_2.weight', 'gmhra.3.ln_2.bias', 'gmhra.3.ln_3.weight', 'gmhra.3.ln_3.bias', 'gmhra.4.dpe.weight', 'gmhra.4.dpe.bias', 'gmhra.4.attn.in_proj_weight', 'gmhra.4.attn.in_proj_bias', 'gmhra.4.attn.out_proj.weight', 'gmhra.4.attn.out_proj.bias', 'gmhra.4.ln_1.weight', 'gmhra.4.ln_1.bias', 'gmhra.4.mlp.c_fc.weight', 'gmhra.4.mlp.c_fc.bias', 'gmhra.4.mlp.c_proj.weight', 'gmhra.4.mlp.c_proj.bias', 'gmhra.4.ln_2.weight', 'gmhra.4.ln_2.bias', 'gmhra.4.ln_3.weight', 'gmhra.4.ln_3.bias', 'gmhra.5.dpe.weight', 'gmhra.5.dpe.bias', 'gmhra.5.attn.in_proj_weight', 'gmhra.5.attn.in_proj_bias', 'gmhra.5.attn.out_proj.weight', 'gmhra.5.attn.out_proj.bias', 'gmhra.5.ln_1.weight', 'gmhra.5.ln_1.bias', 'gmhra.5.mlp.c_fc.weight', 'gmhra.5.mlp.c_fc.bias', 'gmhra.5.mlp.c_proj.weight', 'gmhra.5.mlp.c_proj.bias', 'gmhra.5.ln_2.weight', 'gmhra.5.ln_2.bias', 'gmhra.5.ln_3.weight', 'gmhra.5.ln_3.bias', 'gmhra.6.dpe.weight', 'gmhra.6.dpe.bias', 'gmhra.6.attn.in_proj_weight', 'gmhra.6.attn.in_proj_bias', 'gmhra.6.attn.out_proj.weight', 'gmhra.6.attn.out_proj.bias', 'gmhra.6.ln_1.weight', 'gmhra.6.ln_1.bias', 'gmhra.6.mlp.c_fc.weight', 'gmhra.6.mlp.c_fc.bias', 'gmhra.6.mlp.c_proj.weight', 'gmhra.6.mlp.c_proj.bias', 'gmhra.6.ln_2.weight', 'gmhra.6.ln_2.bias', 'gmhra.6.ln_3.weight', 'gmhra.6.ln_3.bias', 'gmhra.7.dpe.weight', 'gmhra.7.dpe.bias', 'gmhra.7.attn.in_proj_weight', 'gmhra.7.attn.in_proj_bias', 'gmhra.7.attn.out_proj.weight', 'gmhra.7.attn.out_proj.bias', 'gmhra.7.ln_1.weight', 'gmhra.7.ln_1.bias', 'gmhra.7.mlp.c_fc.weight', 'gmhra.7.mlp.c_fc.bias', 'gmhra.7.mlp.c_proj.weight', 'gmhra.7.mlp.c_proj.bias', 'gmhra.7.ln_2.weight', 'gmhra.7.ln_2.bias', 'gmhra.7.ln_3.weight', 'gmhra.7.ln_3.bias'], unexpected_keys=['norm.weight', 'norm.bias', 'head.weight', 'head.bias', 'blocks.39.norm1.weight', 'blocks.39.norm1.bias', 'blocks.39.attn.q_bias', 'blocks.39.attn.v_bias', 'blocks.39.attn.qkv.weight', 'blocks.39.attn.proj.weight', 'blocks.39.attn.proj.bias', 'blocks.39.norm2.weight', 'blocks.39.norm2.bias', 'blocks.39.mlp.fc1.weight', 'blocks.39.mlp.fc1.bias', 'blocks.39.mlp.fc2.weight', 'blocks.39.mlp.fc2.bias'])
freeze vision encoder
open module: ['gmhra_cls_token', 'gmhra.0.dpe.weight', 'gmhra.0.dpe.bias', 'gmhra.0.attn.in_proj_weight', 'gmhra.0.attn.in_proj_bias', 'gmhra.0.attn.out_proj.weight', 'gmhra.0.attn.out_proj.bias', 'gmhra.0.ln_1.weight', 'gmhra.0.ln_1.bias', 'gmhra.0.mlp.c_fc.weight', 'gmhra.0.mlp.c_fc.bias', 'gmhra.0.mlp.c_proj.weight', 'gmhra.0.mlp.c_proj.bias', 'gmhra.0.ln_2.weight', 'gmhra.0.ln_2.bias', 'gmhra.0.ln_3.weight', 'gmhra.0.ln_3.bias', 'gmhra.1.dpe.weight', 'gmhra.1.dpe.bias', 'gmhra.1.attn.in_proj_weight', 'gmhra.1.attn.in_proj_bias', 'gmhra.1.attn.out_proj.weight', 'gmhra.1.attn.out_proj.bias', 'gmhra.1.ln_1.weight', 'gmhra.1.ln_1.bias', 'gmhra.1.mlp.c_fc.weight', 'gmhra.1.mlp.c_fc.bias', 'gmhra.1.mlp.c_proj.weight', 'gmhra.1.mlp.c_proj.bias', 'gmhra.1.ln_2.weight', 'gmhra.1.ln_2.bias', 'gmhra.1.ln_3.weight', 'gmhra.1.ln_3.bias', 'gmhra.2.dpe.weight', 'gmhra.2.dpe.bias', 'gmhra.2.attn.in_proj_weight', 'gmhra.2.attn.in_proj_bias', 'gmhra.2.attn.out_proj.weight', 'gmhra.2.attn.out_proj.bias', 'gmhra.2.ln_1.weight', 'gmhra.2.ln_1.bias', 'gmhra.2.mlp.c_fc.weight', 'gmhra.2.mlp.c_fc.bias', 'gmhra.2.mlp.c_proj.weight', 'gmhra.2.mlp.c_proj.bias', 'gmhra.2.ln_2.weight', 'gmhra.2.ln_2.bias', 'gmhra.2.ln_3.weight', 'gmhra.2.ln_3.bias', 'gmhra.3.dpe.weight', 'gmhra.3.dpe.bias', 'gmhra.3.attn.in_proj_weight', 'gmhra.3.attn.in_proj_bias', 'gmhra.3.attn.out_proj.weight', 'gmhra.3.attn.out_proj.bias', 'gmhra.3.ln_1.weight', 'gmhra.3.ln_1.bias', 'gmhra.3.mlp.c_fc.weight', 'gmhra.3.mlp.c_fc.bias', 'gmhra.3.mlp.c_proj.weight', 'gmhra.3.mlp.c_proj.bias', 'gmhra.3.ln_2.weight', 'gmhra.3.ln_2.bias', 'gmhra.3.ln_3.weight', 'gmhra.3.ln_3.bias', 'gmhra.4.dpe.weight', 'gmhra.4.dpe.bias', 'gmhra.4.attn.in_proj_weight', 'gmhra.4.attn.in_proj_bias', 'gmhra.4.attn.out_proj.weight', 'gmhra.4.attn.out_proj.bias', 'gmhra.4.ln_1.weight', 'gmhra.4.ln_1.bias', 'gmhra.4.mlp.c_fc.weight', 'gmhra.4.mlp.c_fc.bias', 'gmhra.4.mlp.c_proj.weight', 'gmhra.4.mlp.c_proj.bias', 'gmhra.4.ln_2.weight', 'gmhra.4.ln_2.bias', 'gmhra.4.ln_3.weight', 'gmhra.4.ln_3.bias', 'gmhra.5.dpe.weight', 'gmhra.5.dpe.bias', 'gmhra.5.attn.in_proj_weight', 'gmhra.5.attn.in_proj_bias', 'gmhra.5.attn.out_proj.weight', 'gmhra.5.attn.out_proj.bias', 'gmhra.5.ln_1.weight', 'gmhra.5.ln_1.bias', 'gmhra.5.mlp.c_fc.weight', 'gmhra.5.mlp.c_fc.bias', 'gmhra.5.mlp.c_proj.weight', 'gmhra.5.mlp.c_proj.bias', 'gmhra.5.ln_2.weight', 'gmhra.5.ln_2.bias', 'gmhra.5.ln_3.weight', 'gmhra.5.ln_3.bias', 'gmhra.6.dpe.weight', 'gmhra.6.dpe.bias', 'gmhra.6.attn.in_proj_weight', 'gmhra.6.attn.in_proj_bias', 'gmhra.6.attn.out_proj.weight', 'gmhra.6.attn.out_proj.bias', 'gmhra.6.ln_1.weight', 'gmhra.6.ln_1.bias', 'gmhra.6.mlp.c_fc.weight', 'gmhra.6.mlp.c_fc.bias', 'gmhra.6.mlp.c_proj.weight', 'gmhra.6.mlp.c_proj.bias', 'gmhra.6.ln_2.weight', 'gmhra.6.ln_2.bias', 'gmhra.6.ln_3.weight', 'gmhra.6.ln_3.bias', 'gmhra.7.dpe.weight', 'gmhra.7.dpe.bias', 'gmhra.7.attn.in_proj_weight', 'gmhra.7.attn.in_proj_bias', 'gmhra.7.attn.out_proj.weight', 'gmhra.7.attn.out_proj.bias', 'gmhra.7.ln_1.weight', 'gmhra.7.ln_1.bias', 'gmhra.7.mlp.c_fc.weight', 'gmhra.7.mlp.c_fc.bias', 'gmhra.7.mlp.c_proj.weight', 'gmhra.7.mlp.c_proj.bias', 'gmhra.7.ln_2.weight', 'gmhra.7.ln_2.bias', 'gmhra.7.ln_3.weight', 'gmhra.7.ln_3.bias']
open ln_vision
Loading VIT Done
Loading Q-Former
Drop_path:[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
BertConfig {
"add_cross_attention": true,
"architectures": [
"BertForMaskedLM"
],
"attention_probs_dropout_prob": 0.0,
"classifier_dropout": null,
"cross_attention_freq": 2,
"drop_path_list": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"encoder_width": 1408,
"gradient_checkpointing": false,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.0,
"hidden_size": 768,
"initializer_range": 0.02,
"intermediate_size": 3072,
"layer_norm_eps": 1e-12,
"max_position_embeddings": 512,
"model_type": "bert",
"num_attention_heads": 12,
"num_hidden_layers": 12,
"pad_token_id": 0,
"position_embedding_type": "absolute",
"query_length": 32,
"transformers_version": "4.28.1",
"type_vocab_size": 2,
"use_cache": true,
"vocab_size": 30522
}
Load QFormer from D:/GIT_Project/Ask-Anything/Bilp2/blip2_pretrained_flant5xxl.pth
_IncompatibleKeys(missing_keys=['visual_encoder.cls_token', 'visual_encoder.pos_embed', 'visual_encoder.gmhra_cls_token', 'visual_encoder.patch_embed.proj.weight', 'visual_encoder.patch_embed.proj.bias', 'visual_encoder.blocks.0.norm1.weight', 'visual_encoder.blocks.0.norm1.bias', 'visual_encoder.blocks.0.attn.q_bias', 'visual_encoder.blocks.0.attn.v_bias', 'visual_encoder.blocks.0.attn.qkv.weight', 'visual_encoder.blocks.0.attn.proj.weight', 'visual_encoder.blocks.0.attn.proj.bias', 'visual_encoder.blocks.0.norm2.weight', 'visual_encoder.blocks.0.norm2.bias', 'visual_encoder.blocks.0.mlp.fc1.weight', 'visual_encoder.blocks.0.mlp.fc1.bias', 'visual_encoder.blocks.0.mlp.fc2.weight', 'visual_encoder.blocks.0.mlp.fc2.bias', 'visual_encoder.blocks.1.norm1.weight', 'visual_encoder.blocks.1.norm1.bias', 'visual_encoder.blocks.1.attn.q_bias', 'visual_encoder.blocks.1.attn.v_bias', 'visual_encoder.blocks.1.attn.qkv.weight', 'visual_encoder.blocks.1.attn.proj.weight', 'visual_encoder.blocks.1.attn.proj.bias', 'visual_encoder.blocks.1.norm2.weight', 'visual_encoder.blocks.1.norm2.bias', 'visual_encoder.blocks.1.mlp.fc1.weight', 'visual_encoder.blocks.1.mlp.fc1.bias', 'visual_encoder.blocks.1.mlp.fc2.weight', 'visual_encoder.blocks.1.mlp.fc2.bias', 'visual_encoder.blocks.2.norm1.weight', 'visual_encoder.blocks.2.norm1.bias', 'visual_encoder.blocks.2.attn.q_bias', 'visual_encoder.blocks.2.attn.v_bias', 'visual_encoder.blocks.2.attn.qkv.weight', 'visual_encoder.blocks.2.attn.proj.weight', 'visual_encoder.blocks.2.attn.proj.bias', 'visual_encoder.blocks.2.norm2.weight', 'visual_encoder.blocks.2.norm2.bias', 'visual_encoder.blocks.2.mlp.fc1.weight', 'visual_encoder.blocks.2.mlp.fc1.bias', 'visual_encoder.blocks.2.mlp.fc2.weight', 'visual_encoder.blocks.2.mlp.fc2.bias', 'visual_encoder.blocks.3.norm1.weight', 'visual_encoder.blocks.3.norm1.bias', 'visual_encoder.blocks.3.attn.q_bias', 'visual_encoder.blocks.3.attn.v_bias', 'visual_encoder.blocks.3.attn.qkv.weight', 'visual_encoder.blocks.3.attn.proj.weight', 'visual_encoder.blocks.3.attn.proj.bias', 'visual_encoder.blocks.3.norm2.weight', 'visual_encoder.blocks.3.norm2.bias', 'visual_encoder.blocks.3.mlp.fc1.weight', 'visual_encoder.blocks.3.mlp.fc1.bias', 'visual_encoder.blocks.3.mlp.fc2.weight', 'visual_encoder.blocks.3.mlp.fc2.bias', 'visual_encoder.blocks.4.norm1.weight', 'visual_encoder.blocks.4.norm1.bias', 'visual_encoder.blocks.4.attn.q_bias', 'visual_encoder.blocks.4.attn.v_bias', 'visual_encoder.blocks.4.attn.qkv.weight', 'visual_encoder.blocks.4.attn.proj.weight', 'visual_encoder.blocks.4.attn.proj.bias', 'visual_encoder.blocks.4.norm2.weight', 'visual_encoder.blocks.4.norm2.bias', 'visual_encoder.blocks.4.mlp.fc1.weight', 'visual_encoder.blocks.4.mlp.fc1.bias', 'visual_encoder.blocks.4.mlp.fc2.weight', 'visual_encoder.blocks.4.mlp.fc2.bias', 'visual_encoder.blocks.5.norm1.weight', 'visual_encoder.blocks.5.norm1.bias', 'visual_encoder.blocks.5.attn.q_bias', 'visual_encoder.blocks.5.attn.v_bias', 'visual_encoder.blocks.5.attn.qkv.weight', 'visual_encoder.blocks.5.attn.proj.weight', 'visual_encoder.blocks.5.attn.proj.bias', 'visual_encoder.blocks.5.norm2.weight', 'visual_encoder.blocks.5.norm2.bias', 'visual_encoder.blocks.5.mlp.fc1.weight', 'visual_encoder.blocks.5.mlp.fc1.bias', 'visual_encoder.blocks.5.mlp.fc2.weight', 'visual_encoder.blocks.5.mlp.fc2.bias', 'visual_encoder.blocks.6.norm1.weight', 'visual_encoder.blocks.6.norm1.bias', 'visual_encoder.blocks.6.attn.q_bias', 'visual_encoder.blocks.6.attn.v_bias', 'visual_encoder.blocks.6.attn.qkv.weight', 'visual_encoder.blocks.6.attn.proj.weight', 'visual_encoder.blocks.6.attn.proj.bias', 'visual_encoder.blocks.6.norm2.weight', 'visual_encoder.blocks.6.norm2.bias', 'visual_encoder.blocks.6.mlp.fc1.weight', 'visual_encoder.blocks.6.mlp.fc1.bias', 'visual_encoder.blocks.6.mlp.fc2.weight', 'visual_encoder.blocks.6.mlp.fc2.bias', 'visual_encoder.blocks.7.norm1.weight', 'visual_encoder.blocks.7.norm1.bias', 'visual_encoder.blocks.7.attn.q_bias', 'visual_encoder.blocks.7.attn.v_bias', 'visual_encoder.blocks.7.attn.qkv.weight', 'visual_encoder.blocks.7.attn.proj.weight', 'visual_encoder.blocks.7.attn.proj.bias', 'visual_encoder.blocks.7.norm2.weight', 'visual_encoder.blocks.7.norm2.bias', 'visual_encoder.blocks.7.mlp.fc1.weight', 'visual_encoder.blocks.7.mlp.fc1.bias', 'visual_encoder.blocks.7.mlp.fc2.weight', 'visual_encoder.blocks.7.mlp.fc2.bias', 'visual_encoder.blocks.8.norm1.weight', 'visual_encoder.blocks.8.norm1.bias', 'visual_encoder.blocks.8.attn.q_bias', 'visual_encoder.blocks.8.attn.v_bias', 'visual_encoder.blocks.8.attn.qkv.weight', 'visual_encoder.blocks.8.attn.proj.weight', 'visual_encoder.blocks.8.attn.proj.bias', 'visual_encoder.blocks.8.norm2.weight', 'visual_encoder.blocks.8.norm2.bias', 'visual_encoder.blocks.8.mlp.fc1.weight', 'visual_encoder.blocks.8.mlp.fc1.bias', 'visual_encoder.blocks.8.mlp.fc2.weight', 'visual_encoder.blocks.8.mlp.fc2.bias', 'visual_encoder.blocks.9.norm1.weight', 'visual_encoder.blocks.9.norm1.bias', 'visual_encoder.blocks.9.attn.q_bias', 'visual_encoder.blocks.9.attn.v_bias', 'visual_encoder.blocks.9.attn.qkv.weight', 'visual_encoder.blocks.9.attn.proj.weight', 'visual_encoder.blocks.9.attn.proj.bias', 'visual_encoder.blocks.9.norm2.weight', 'visual_encoder.blocks.9.norm2.bias', 'visual_encoder.blocks.9.mlp.fc1.weight', 'visual_encoder.blocks.9.mlp.fc1.bias', 'visual_encoder.blocks.9.mlp.fc2.weight', 'visual_encoder.blocks.9.mlp.fc2.bias', 'visual_encoder.blocks.10.norm1.weight', 'visual_encoder.blocks.10.norm1.bias', 'visual_encoder.blocks.10.attn.q_bias', 'visual_encoder.blocks.10.attn.v_bias', 'visual_encoder.blocks.10.attn.qkv.weight', 'visual_encoder.blocks.10.attn.proj.weight', 'visual_encoder.blocks.10.attn.proj.bias', 'visual_encoder.blocks.10.norm2.weight', 'visual_encoder.blocks.10.norm2.bias', 'visual_encoder.blocks.10.mlp.fc1.weight', 'visual_encoder.blocks.10.mlp.fc1.bias', 'visual_encoder.blocks.10.mlp.fc2.weight', 'visual_encoder.blocks.10.mlp.fc2.bias', 'visual_encoder.blocks.11.norm1.weight', 'visual_encoder.blocks.11.norm1.bias', 'visual_encoder.blocks.11.attn.q_bias', 'visual_encoder.blocks.11.attn.v_bias', 'visual_encoder.blocks.11.attn.qkv.weight', 'visual_encoder.blocks.11.attn.proj.weight', 'visual_encoder.blocks.11.attn.proj.bias', 'visual_encoder.blocks.11.norm2.weight', 'visual_encoder.blocks.11.norm2.bias', 'visual_encoder.blocks.11.mlp.fc1.weight', 'visual_encoder.blocks.11.mlp.fc1.bias', 'visual_encoder.blocks.11.mlp.fc2.weight', 'visual_encoder.blocks.11.mlp.fc2.bias', 'visual_encoder.blocks.12.norm1.weight', 'visual_encoder.blocks.12.norm1.bias', 'visual_encoder.blocks.12.attn.q_bias', 'visual_encoder.blocks.12.attn.v_bias', 'visual_encoder.blocks.12.attn.qkv.weight', 'visual_encoder.blocks.12.attn.proj.weight', 'visual_encoder.blocks.12.attn.proj.bias', 'visual_encoder.blocks.12.norm2.weight', 'visual_encoder.blocks.12.norm2.bias', 'visual_encoder.blocks.12.mlp.fc1.weight', 'visual_encoder.blocks.12.mlp.fc1.bias', 'visual_encoder.blocks.12.mlp.fc2.weight', 'visual_encoder.blocks.12.mlp.fc2.bias', 'visual_encoder.blocks.13.norm1.weight', 'visual_encoder.blocks.13.norm1.bias', 'visual_encoder.blocks.13.attn.q_bias', 'visual_encoder.blocks.13.attn.v_bias', 'visual_encoder.blocks.13.attn.qkv.weight', 'visual_encoder.blocks.13.attn.proj.weight', 'visual_encoder.blocks.13.attn.proj.bias', 'visual_encoder.blocks.13.norm2.weight', 'visual_encoder.blocks.13.norm2.bias', 'visual_encoder.blocks.13.mlp.fc1.weight', 'visual_encoder.blocks.13.mlp.fc1.bias', 'visual_encoder.blocks.13.mlp.fc2.weight', 'visual_encoder.blocks.13.mlp.fc2.bias', 'visual_encoder.blocks.14.norm1.weight', 'visual_encoder.blocks.14.norm1.bias', 'visual_encoder.blocks.14.attn.q_bias', 'visual_encoder.blocks.14.attn.v_bias', 'visual_encoder.blocks.14.attn.qkv.weight', 'visual_encoder.blocks.14.attn.proj.weight', 'visual_encoder.blocks.14.attn.proj.bias', 'visual_encoder.blocks.14.norm2.weight', 'visual_encoder.blocks.14.norm2.bias', 'visual_encoder.blocks.14.mlp.fc1.weight', 'visual_encoder.blocks.14.mlp.fc1.bias', 'visual_encoder.blocks.14.mlp.fc2.weight', 'visual_encoder.blocks.14.mlp.fc2.bias', 'visual_encoder.blocks.15.norm1.weight', 'visual_encoder.blocks.15.norm1.bias', 'visual_encoder.blocks.15.attn.q_bias', 'visual_encoder.blocks.15.attn.v_bias', 'visual_encoder.blocks.15.attn.qkv.weight', 'visual_encoder.blocks.15.attn.proj.weight', 'visual_encoder.blocks.15.attn.proj.bias', 'visual_encoder.blocks.15.norm2.weight', 'visual_encoder.blocks.15.norm2.bias', 'visual_encoder.blocks.15.mlp.fc1.weight', 'visual_encoder.blocks.15.mlp.fc1.bias', 'visual_encoder.blocks.15.mlp.fc2.weight', 'visual_encoder.blocks.15.mlp.fc2.bias', 'visual_encoder.blocks.16.norm1.weight', 'visual_encoder.blocks.16.norm1.bias', 'visual_encoder.blocks.16.attn.q_bias', 'visual_encoder.blocks.16.attn.v_bias', 'visual_encoder.blocks.16.attn.qkv.weight', 'visual_encoder.blocks.16.attn.proj.weight', 'visual_encoder.blocks.16.attn.proj.bias', 'visual_encoder.blocks.16.norm2.weight', 'visual_encoder.blocks.16.norm2.bias', 'visual_encoder.blocks.16.mlp.fc1.weight', 'visual_encoder.blocks.16.mlp.fc1.bias', 'visual_encoder.blocks.16.mlp.fc2.weight', 'visual_encoder.blocks.16.mlp.fc2.bias', 'visual_encoder.blocks.17.norm1.weight', 'visual_encoder.blocks.17.norm1.bias', 'visual_encoder.blocks.17.attn.q_bias', 'visual_encoder.blocks.17.attn.v_bias', 'visual_encoder.blocks.17.attn.qkv.weight', 'visual_encoder.blocks.17.attn.proj.weight', 'visual_encoder.blocks.17.attn.proj.bias', 'visual_encoder.blocks.17.norm2.weight', 'visual_encoder.blocks.17.norm2.bias', 'visual_encoder.blocks.17.mlp.fc1.weight', 'visual_encoder.blocks.17.mlp.fc1.bias', 'visual_encoder.blocks.17.mlp.fc2.weight', 'visual_encoder.blocks.17.mlp.fc2.bias', 'visual_encoder.blocks.18.norm1.weight', 'visual_encoder.blocks.18.norm1.bias', 'visual_encoder.blocks.18.attn.q_bias', 'visual_encoder.blocks.18.attn.v_bias', 'visual_encoder.blocks.18.attn.qkv.weight', 'visual_encoder.blocks.18.attn.proj.weight', 'visual_encoder.blocks.18.attn.proj.bias', 'visual_encoder.blocks.18.norm2.weight', 'visual_encoder.blocks.18.norm2.bias', 'visual_encoder.blocks.18.mlp.fc1.weight', 'visual_encoder.blocks.18.mlp.fc1.bias', 'visual_encoder.blocks.18.mlp.fc2.weight', 'visual_encoder.blocks.18.mlp.fc2.bias', 'visual_encoder.blocks.19.norm1.weight', 'visual_encoder.blocks.19.norm1.bias', 'visual_encoder.blocks.19.attn.q_bias', 'visual_encoder.blocks.19.attn.v_bias', 'visual_encoder.blocks.19.attn.qkv.weight', 'visual_encoder.blocks.19.attn.proj.weight', 'visual_encoder.blocks.19.attn.proj.bias', 'visual_encoder.blocks.19.norm2.weight', 'visual_encoder.blocks.19.norm2.bias', 'visual_encoder.blocks.19.mlp.fc1.weight', 'visual_encoder.blocks.19.mlp.fc1.bias', 'visual_encoder.blocks.19.mlp.fc2.weight', 'visual_encoder.blocks.19.mlp.fc2.bias', 'visual_encoder.blocks.20.norm1.weight', 'visual_encoder.blocks.20.norm1.bias', 'visual_encoder.blocks.20.attn.q_bias', 'visual_encoder.blocks.20.attn.v_bias', 'visual_encoder.blocks.20.attn.qkv.weight', 'visual_encoder.blocks.20.attn.proj.weight', 'visual_encoder.blocks.20.attn.proj.bias', 'visual_encoder.blocks.20.norm2.weight', 'visual_encoder.blocks.20.norm2.bias', 'visual_encoder.blocks.20.mlp.fc1.weight', 'visual_encoder.blocks.20.mlp.fc1.bias', 'visual_encoder.blocks.20.mlp.fc2.weight', 'visual_encoder.blocks.20.mlp.fc2.bias', 'visual_encoder.blocks.21.norm1.weight', 'visual_encoder.blocks.21.norm1.bias', 'visual_encoder.blocks.21.attn.q_bias', 'visual_encoder.blocks.21.attn.v_bias', 'visual_encoder.blocks.21.attn.qkv.weight', 'visual_encoder.blocks.21.attn.proj.weight', 'visual_encoder.blocks.21.attn.proj.bias', 'visual_encoder.blocks.21.norm2.weight', 'visual_encoder.blocks.21.norm2.bias', 'visual_encoder.blocks.21.mlp.fc1.weight', 'visual_encoder.blocks.21.mlp.fc1.bias', 'visual_encoder.blocks.21.mlp.fc2.weight', 'visual_encoder.blocks.21.mlp.fc2.bias', 'visual_encoder.blocks.22.norm1.weight', 'visual_encoder.blocks.22.norm1.bias', 'visual_encoder.blocks.22.attn.q_bias', 'visual_encoder.blocks.22.attn.v_bias', 'visual_encoder.blocks.22.attn.qkv.weight', 'visual_encoder.blocks.22.attn.proj.weight', 'visual_encoder.blocks.22.attn.proj.bias', 'visual_encoder.blocks.22.norm2.weight', 'visual_encoder.blocks.22.norm2.bias', 'visual_encoder.blocks.22.mlp.fc1.weight', 'visual_encoder.blocks.22.mlp.fc1.bias', 'visual_encoder.blocks.22.mlp.fc2.weight', 'visual_encoder.blocks.22.mlp.fc2.bias', 'visual_encoder.blocks.23.norm1.weight', 'visual_encoder.blocks.23.norm1.bias', 'visual_encoder.blocks.23.attn.q_bias', 'visual_encoder.blocks.23.attn.v_bias', 'visual_encoder.blocks.23.attn.qkv.weight', 'visual_encoder.blocks.23.attn.proj.weight', 'visual_encoder.blocks.23.attn.proj.bias', 'visual_encoder.blocks.23.norm2.weight', 'visual_encoder.blocks.23.norm2.bias', 'visual_encoder.blocks.23.mlp.fc1.weight', 'visual_encoder.blocks.23.mlp.fc1.bias', 'visual_encoder.blocks.23.mlp.fc2.weight', 'visual_encoder.blocks.23.mlp.fc2.bias', 'visual_encoder.blocks.24.norm1.weight', 'visual_encoder.blocks.24.norm1.bias', 'visual_encoder.blocks.24.attn.q_bias', 'visual_encoder.blocks.24.attn.v_bias', 'visual_encoder.blocks.24.attn.qkv.weight', 'visual_encoder.blocks.24.attn.proj.weight', 'visual_encoder.blocks.24.attn.proj.bias', 'visual_encoder.blocks.24.norm2.weight', 'visual_encoder.blocks.24.norm2.bias', 'visual_encoder.blocks.24.mlp.fc1.weight', 'visual_encoder.blocks.24.mlp.fc1.bias', 'visual_encoder.blocks.24.mlp.fc2.weight', 'visual_encoder.blocks.24.mlp.fc2.bias', 'visual_encoder.blocks.25.norm1.weight', 'visual_encoder.blocks.25.norm1.bias', 'visual_encoder.blocks.25.attn.q_bias', 'visual_encoder.blocks.25.attn.v_bias', 'visual_encoder.blocks.25.attn.qkv.weight', 'visual_encoder.blocks.25.attn.proj.weight', 'visual_encoder.blocks.25.attn.proj.bias', 'visual_encoder.blocks.25.norm2.weight', 'visual_encoder.blocks.25.norm2.bias', 'visual_encoder.blocks.25.mlp.fc1.weight', 'visual_encoder.blocks.25.mlp.fc1.bias', 'visual_encoder.blocks.25.mlp.fc2.weight', 'visual_encoder.blocks.25.mlp.fc2.bias', 'visual_encoder.blocks.26.norm1.weight', 'visual_encoder.blocks.26.norm1.bias', 'visual_encoder.blocks.26.attn.q_bias', 'visual_encoder.blocks.26.attn.v_bias', 'visual_encoder.blocks.26.attn.qkv.weight', 'visual_encoder.blocks.26.attn.proj.weight', 'visual_encoder.blocks.26.attn.proj.bias', 'visual_encoder.blocks.26.norm2.weight', 'visual_encoder.blocks.26.norm2.bias', 'visual_encoder.blocks.26.mlp.fc1.weight', 'visual_encoder.blocks.26.mlp.fc1.bias', 'visual_encoder.blocks.26.mlp.fc2.weight', 'visual_encoder.blocks.26.mlp.fc2.bias', 'visual_encoder.blocks.27.norm1.weight', 'visual_encoder.blocks.27.norm1.bias', 'visual_encoder.blocks.27.attn.q_bias', 'visual_encoder.blocks.27.attn.v_bias', 'visual_encoder.blocks.27.attn.qkv.weight', 'visual_encoder.blocks.27.attn.proj.weight', 'visual_encoder.blocks.27.attn.proj.bias', 'visual_encoder.blocks.27.norm2.weight', 'visual_encoder.blocks.27.norm2.bias', 'visual_encoder.blocks.27.mlp.fc1.weight', 'visual_encoder.blocks.27.mlp.fc1.bias', 'visual_encoder.blocks.27.mlp.fc2.weight', 'visual_encoder.blocks.27.mlp.fc2.bias', 'visual_encoder.blocks.28.norm1.weight', 'visual_encoder.blocks.28.norm1.bias', 'visual_encoder.blocks.28.attn.q_bias', 'visual_encoder.blocks.28.attn.v_bias', 'visual_encoder.blocks.28.attn.qkv.weight', 'visual_encoder.blocks.28.attn.proj.weight', 'visual_encoder.blocks.28.attn.proj.bias', 'visual_encoder.blocks.28.norm2.weight', 'visual_encoder.blocks.28.norm2.bias', 'visual_encoder.blocks.28.mlp.fc1.weight', 'visual_encoder.blocks.28.mlp.fc1.bias', 'visual_encoder.blocks.28.mlp.fc2.weight', 'visual_encoder.blocks.28.mlp.fc2.bias', 'visual_encoder.blocks.29.norm1.weight', 'visual_encoder.blocks.29.norm1.bias', 'visual_encoder.blocks.29.attn.q_bias', 'visual_encoder.blocks.29.attn.v_bias', 'visual_encoder.blocks.29.attn.qkv.weight', 'visual_encoder.blocks.29.attn.proj.weight', 'visual_encoder.blocks.29.attn.proj.bias', 'visual_encoder.blocks.29.norm2.weight', 'visual_encoder.blocks.29.norm2.bias', 'visual_encoder.blocks.29.mlp.fc1.weight', 'visual_encoder.blocks.29.mlp.fc1.bias', 'visual_encoder.blocks.29.mlp.fc2.weight', 'visual_encoder.blocks.29.mlp.fc2.bias', 'visual_encoder.blocks.30.norm1.weight', 'visual_encoder.blocks.30.norm1.bias', 'visual_encoder.blocks.30.attn.q_bias', 'visual_encoder.blocks.30.attn.v_bias', 'visual_encoder.blocks.30.attn.qkv.weight', 'visual_encoder.blocks.30.attn.proj.weight', 'visual_encoder.blocks.30.attn.proj.bias', 'visual_encoder.blocks.30.norm2.weight', 'visual_encoder.blocks.30.norm2.bias', 'visual_encoder.blocks.30.mlp.fc1.weight', 'visual_encoder.blocks.30.mlp.fc1.bias', 'visual_encoder.blocks.30.mlp.fc2.weight', 'visual_encoder.blocks.30.mlp.fc2.bias', 'visual_encoder.blocks.31.norm1.weight', 'visual_encoder.blocks.31.norm1.bias', 'visual_encoder.blocks.31.attn.q_bias', 'visual_encoder.blocks.31.attn.v_bias', 'visual_encoder.blocks.31.attn.qkv.weight', 'visual_encoder.blocks.31.attn.proj.weight', 'visual_encoder.blocks.31.attn.proj.bias', 'visual_encoder.blocks.31.norm2.weight', 'visual_encoder.blocks.31.norm2.bias', 'visual_encoder.blocks.31.mlp.fc1.weight', 'visual_encoder.blocks.31.mlp.fc1.bias', 'visual_encoder.blocks.31.mlp.fc2.weight', 'visual_encoder.blocks.31.mlp.fc2.bias', 'visual_encoder.blocks.32.norm1.weight', 'visual_encoder.blocks.32.norm1.bias', 'visual_encoder.blocks.32.attn.q_bias', 'visual_encoder.blocks.32.attn.v_bias', 'visual_encoder.blocks.32.attn.qkv.weight', 'visual_encoder.blocks.32.attn.proj.weight', 'visual_encoder.blocks.32.attn.proj.bias', 'visual_encoder.blocks.32.norm2.weight', 'visual_encoder.blocks.32.norm2.bias', 'visual_encoder.blocks.32.mlp.fc1.weight', 'visual_encoder.blocks.32.mlp.fc1.bias', 'visual_encoder.blocks.32.mlp.fc2.weight', 'visual_encoder.blocks.32.mlp.fc2.bias', 'visual_encoder.blocks.33.norm1.weight', 'visual_encoder.blocks.33.norm1.bias', 'visual_encoder.blocks.33.attn.q_bias', 'visual_encoder.blocks.33.attn.v_bias', 'visual_encoder.blocks.33.attn.qkv.weight', 'visual_encoder.blocks.33.attn.proj.weight', 'visual_encoder.blocks.33.attn.proj.bias', 'visual_encoder.blocks.33.norm2.weight', 'visual_encoder.blocks.33.norm2.bias', 'visual_encoder.blocks.33.mlp.fc1.weight', 'visual_encoder.blocks.33.mlp.fc1.bias', 'visual_encoder.blocks.33.mlp.fc2.weight', 'visual_encoder.blocks.33.mlp.fc2.bias', 'visual_encoder.blocks.34.norm1.weight', 'visual_encoder.blocks.34.norm1.bias', 'visual_encoder.blocks.34.attn.q_bias', 'visual_encoder.blocks.34.attn.v_bias', 'visual_encoder.blocks.34.attn.qkv.weight', 'visual_encoder.blocks.34.attn.proj.weight', 'visual_encoder.blocks.34.attn.proj.bias', 'visual_encoder.blocks.34.norm2.weight', 'visual_encoder.blocks.34.norm2.bias', 'visual_encoder.blocks.34.mlp.fc1.weight', 'visual_encoder.blocks.34.mlp.fc1.bias', 'visual_encoder.blocks.34.mlp.fc2.weight', 'visual_encoder.blocks.34.mlp.fc2.bias', 'visual_encoder.blocks.35.norm1.weight', 'visual_encoder.blocks.35.norm1.bias', 'visual_encoder.blocks.35.attn.q_bias', 'visual_encoder.blocks.35.attn.v_bias', 'visual_encoder.blocks.35.attn.qkv.weight', 'visual_encoder.blocks.35.attn.proj.weight', 'visual_encoder.blocks.35.attn.proj.bias', 'visual_encoder.blocks.35.norm2.weight', 'visual_encoder.blocks.35.norm2.bias', 'visual_encoder.blocks.35.mlp.fc1.weight', 'visual_encoder.blocks.35.mlp.fc1.bias', 'visual_encoder.blocks.35.mlp.fc2.weight', 'visual_encoder.blocks.35.mlp.fc2.bias', 'visual_encoder.blocks.36.norm1.weight', 'visual_encoder.blocks.36.norm1.bias', 'visual_encoder.blocks.36.attn.q_bias', 'visual_encoder.blocks.36.attn.v_bias', 'visual_encoder.blocks.36.attn.qkv.weight', 'visual_encoder.blocks.36.attn.proj.weight', 'visual_encoder.blocks.36.attn.proj.bias', 'visual_encoder.blocks.36.norm2.weight', 'visual_encoder.blocks.36.norm2.bias', 'visual_encoder.blocks.36.mlp.fc1.weight', 'visual_encoder.blocks.36.mlp.fc1.bias', 'visual_encoder.blocks.36.mlp.fc2.weight', 'visual_encoder.blocks.36.mlp.fc2.bias', 'visual_encoder.blocks.37.norm1.weight', 'visual_encoder.blocks.37.norm1.bias', 'visual_encoder.blocks.37.attn.q_bias', 'visual_encoder.blocks.37.attn.v_bias', 'visual_encoder.blocks.37.attn.qkv.weight', 'visual_encoder.blocks.37.attn.proj.weight', 'visual_encoder.blocks.37.attn.proj.bias', 'visual_encoder.blocks.37.norm2.weight', 'visual_encoder.blocks.37.norm2.bias', 'visual_encoder.blocks.37.mlp.fc1.weight', 'visual_encoder.blocks.37.mlp.fc1.bias', 'visual_encoder.blocks.37.mlp.fc2.weight', 'visual_encoder.blocks.37.mlp.fc2.bias', 'visual_encoder.blocks.38.norm1.weight', 'visual_encoder.blocks.38.norm1.bias', 'visual_encoder.blocks.38.attn.q_bias', 'visual_encoder.blocks.38.attn.v_bias', 'visual_encoder.blocks.38.attn.qkv.weight', 'visual_encoder.blocks.38.attn.proj.weight', 'visual_encoder.blocks.38.attn.proj.bias', 'visual_encoder.blocks.38.norm2.weight', 'visual_encoder.blocks.38.norm2.bias', 'visual_encoder.blocks.38.mlp.fc1.weight', 'visual_encoder.blocks.38.mlp.fc1.bias', 'visual_encoder.blocks.38.mlp.fc2.weight', 'visual_encoder.blocks.38.mlp.fc2.bias', 'visual_encoder.gmhra.0.dpe.weight', 'visual_encoder.gmhra.0.dpe.bias', 'visual_encoder.gmhra.0.attn.in_proj_weight', 'visual_encoder.gmhra.0.attn.in_proj_bias', 'visual_encoder.gmhra.0.attn.out_proj.weight', 'visual_encoder.gmhra.0.attn.out_proj.bias', 'visual_encoder.gmhra.0.ln_1.weight', 'visual_encoder.gmhra.0.ln_1.bias', 'visual_encoder.gmhra.0.mlp.c_fc.weight', 'visual_encoder.gmhra.0.mlp.c_fc.bias', 'visual_encoder.gmhra.0.mlp.c_proj.weight', 'visual_encoder.gmhra.0.mlp.c_proj.bias', 'visual_encoder.gmhra.0.ln_2.weight', 'visual_encoder.gmhra.0.ln_2.bias', 'visual_encoder.gmhra.0.ln_3.weight', 'visual_encoder.gmhra.0.ln_3.bias', 'visual_encoder.gmhra.1.dpe.weight', 'visual_encoder.gmhra.1.dpe.bias', 'visual_encoder.gmhra.1.attn.in_proj_weight', 'visual_encoder.gmhra.1.attn.in_proj_bias', 'visual_encoder.gmhra.1.attn.out_proj.weight', 'visual_encoder.gmhra.1.attn.out_proj.bias', 'visual_encoder.gmhra.1.ln_1.weight', 'visual_encoder.gmhra.1.ln_1.bias', 'visual_encoder.gmhra.1.mlp.c_fc.weight', 'visual_encoder.gmhra.1.mlp.c_fc.bias', 'visual_encoder.gmhra.1.mlp.c_proj.weight', 'visual_encoder.gmhra.1.mlp.c_proj.bias', 'visual_encoder.gmhra.1.ln_2.weight', 'visual_encoder.gmhra.1.ln_2.bias', 'visual_encoder.gmhra.1.ln_3.weight', 'visual_encoder.gmhra.1.ln_3.bias', 'visual_encoder.gmhra.2.dpe.weight', 'visual_encoder.gmhra.2.dpe.bias', 'visual_encoder.gmhra.2.attn.in_proj_weight', 'visual_encoder.gmhra.2.attn.in_proj_bias', 'visual_encoder.gmhra.2.attn.out_proj.weight', 'visual_encoder.gmhra.2.attn.out_proj.bias', 'visual_encoder.gmhra.2.ln_1.weight', 'visual_encoder.gmhra.2.ln_1.bias', 'visual_encoder.gmhra.2.mlp.c_fc.weight', 'visual_encoder.gmhra.2.mlp.c_fc.bias', 'visual_encoder.gmhra.2.mlp.c_proj.weight', 'visual_encoder.gmhra.2.mlp.c_proj.bias', 'visual_encoder.gmhra.2.ln_2.weight', 'visual_encoder.gmhra.2.ln_2.bias', 'visual_encoder.gmhra.2.ln_3.weight', 'visual_encoder.gmhra.2.ln_3.bias', 'visual_encoder.gmhra.3.dpe.weight', 'visual_encoder.gmhra.3.dpe.bias', 'visual_encoder.gmhra.3.attn.in_proj_weight', 'visual_encoder.gmhra.3.attn.in_proj_bias', 'visual_encoder.gmhra.3.attn.out_proj.weight', 'visual_encoder.gmhra.3.attn.out_proj.bias', 'visual_encoder.gmhra.3.ln_1.weight', 'visual_encoder.gmhra.3.ln_1.bias', 'visual_encoder.gmhra.3.mlp.c_fc.weight', 'visual_encoder.gmhra.3.mlp.c_fc.bias', 'visual_encoder.gmhra.3.mlp.c_proj.weight', 'visual_encoder.gmhra.3.mlp.c_proj.bias', 'visual_encoder.gmhra.3.ln_2.weight', 'visual_encoder.gmhra.3.ln_2.bias', 'visual_encoder.gmhra.3.ln_3.weight', 'visual_encoder.gmhra.3.ln_3.bias', 'visual_encoder.gmhra.4.dpe.weight', 'visual_encoder.gmhra.4.dpe.bias', 'visual_encoder.gmhra.4.attn.in_proj_weight', 'visual_encoder.gmhra.4.attn.in_proj_bias', 'visual_encoder.gmhra.4.attn.out_proj.weight', 'visual_encoder.gmhra.4.attn.out_proj.bias', 'visual_encoder.gmhra.4.ln_1.weight', 'visual_encoder.gmhra.4.ln_1.bias', 'visual_encoder.gmhra.4.mlp.c_fc.weight', 'visual_encoder.gmhra.4.mlp.c_fc.bias', 'visual_encoder.gmhra.4.mlp.c_proj.weight', 'visual_encoder.gmhra.4.mlp.c_proj.bias', 'visual_encoder.gmhra.4.ln_2.weight', 'visual_encoder.gmhra.4.ln_2.bias', 'visual_encoder.gmhra.4.ln_3.weight', 'visual_encoder.gmhra.4.ln_3.bias', 'visual_encoder.gmhra.5.dpe.weight', 'visual_encoder.gmhra.5.dpe.bias', 'visual_encoder.gmhra.5.attn.in_proj_weight', 'visual_encoder.gmhra.5.attn.in_proj_bias', 'visual_encoder.gmhra.5.attn.out_proj.weight', 'visual_encoder.gmhra.5.attn.out_proj.bias', 'visual_encoder.gmhra.5.ln_1.weight', 'visual_encoder.gmhra.5.ln_1.bias', 'visual_encoder.gmhra.5.mlp.c_fc.weight', 'visual_encoder.gmhra.5.mlp.c_fc.bias', 'visual_encoder.gmhra.5.mlp.c_proj.weight', 'visual_encoder.gmhra.5.mlp.c_proj.bias', 'visual_encoder.gmhra.5.ln_2.weight', 'visual_encoder.gmhra.5.ln_2.bias', 'visual_encoder.gmhra.5.ln_3.weight', 'visual_encoder.gmhra.5.ln_3.bias', 'visual_encoder.gmhra.6.dpe.weight', 'visual_encoder.gmhra.6.dpe.bias', 'visual_encoder.gmhra.6.attn.in_proj_weight', 'visual_encoder.gmhra.6.attn.in_proj_bias', 'visual_encoder.gmhra.6.attn.out_proj.weight', 'visual_encoder.gmhra.6.attn.out_proj.bias', 'visual_encoder.gmhra.6.ln_1.weight', 'visual_encoder.gmhra.6.ln_1.bias', 'visual_encoder.gmhra.6.mlp.c_fc.weight', 'visual_encoder.gmhra.6.mlp.c_fc.bias', 'visual_encoder.gmhra.6.mlp.c_proj.weight', 'visual_encoder.gmhra.6.mlp.c_proj.bias', 'visual_encoder.gmhra.6.ln_2.weight', 'visual_encoder.gmhra.6.ln_2.bias', 'visual_encoder.gmhra.6.ln_3.weight', 'visual_encoder.gmhra.6.ln_3.bias', 'visual_encoder.gmhra.7.dpe.weight', 'visual_encoder.gmhra.7.dpe.bias', 'visual_encoder.gmhra.7.attn.in_proj_weight', 'visual_encoder.gmhra.7.attn.in_proj_bias', 'visual_encoder.gmhra.7.attn.out_proj.weight', 'visual_encoder.gmhra.7.attn.out_proj.bias', 'visual_encoder.gmhra.7.ln_1.weight', 'visual_encoder.gmhra.7.ln_1.bias', 'visual_encoder.gmhra.7.mlp.c_fc.weight', 'visual_encoder.gmhra.7.mlp.c_fc.bias', 'visual_encoder.gmhra.7.mlp.c_proj.weight', 'visual_encoder.gmhra.7.mlp.c_proj.bias', 'visual_encoder.gmhra.7.ln_2.weight', 'visual_encoder.gmhra.7.ln_2.bias', 'visual_encoder.gmhra.7.ln_3.weight', 'visual_encoder.gmhra.7.ln_3.bias'], unexpected_keys=['t5_proj.weight', 't5_proj.bias'])
Add extra 64 tokens in QFormer
freeze Qformer
Loading Q-Former Done
Loading LLAMA