When trying to run bash scripts/perfect.sh
, training gets stuck somewhere in a Linear layer of PyTorch.
For context, I'm for now only interested in yielding some result training only for SST-5, even if it is totally useless, so I've reduced some parameters to lower values to reduce computing time on a laptop with limited resources. They are as follows (defined in configs/perfect.json
:
{
"task": "sst-5",
"max_seq_length": 256,
"per_device_train_batch_size": 2,
"K": 8,
"data_dir": "datasets_processed",
"output_dir": "outputs",
"model_name_or_path": "roberta-large",
"do_train": true,
"do_eval": true,
"do_predict": true,
"learning_rate": 1e-4,
"max_steps": 4,
"eval_steps": 3,
"save_steps": 3,
"data_seed": 50,
"seed": 1,
"load_best_model_at_end": true,
"metric_for_best_model": "average",
"greater_is_better": true,
"evaluation_strategy": "steps",
"save_strategy": "steps",
"save_total_limit": 1,
"overwrite_output_dir": true,
"soft_pet": true,
"soft_pet_loss": "extra_tokens",
"extra_without_original": true,
"extra_tokens_init": "random",
"soft_mask_labels_learning_rate": 1e-1,
"adapter_tune": true,
"tune_layernorms": true,
"add_layer_norm_after_adapter": false,
"add_layer_norm_before_adapter": false,
"train_in_batch": true,
"add_adapter_after_attention": false,
"add_adapter_after_feedforward": true,
"extra_embd_initializer_range": 1e-4,
"overwrite_cache": true,
"per_device_eval_batch_size": 10,
"prototypical_eval": true,
"eval_soft_pet_aggregation": "max",
"prototypical_similarity": "euc",
"token_hinge_loss": true,
"mask_position": "1"
}
The training procedure starts and gets stuck at step three out of four, first yielding the WARNING:
WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/timdadum/.cache/huggingface/datasets/json/default-9463d57d37ff152e/0.0.0/c2d554c3377ea79c7664b93dc65d0803b45e3279000f993c7bfd18937fd7f426/cache-91b7584a2265b1f5.arrow
Then, training gets stuck. Upon aborting, the following traceback is present:
Traceback (most recent call last):
File "run_clm.py", line 516, in
main()
File "run_clm.py", line 426, in main
train_result = trainer.train(resume_from_checkpoint=checkpoint)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/transformers/trainer.py", line 1340, in train
self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/transformers/trainer.py", line 1445, in _maybe_log_save_evaluate
metrics = self.evaluate(ignore_keys=ignore_keys_for_eval)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 133, in evaluate
output = self.eval_loop(
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 185, in eval_loop
metrics = self.compute_pet_metrics(eval_datasets, model, self.extra_info[metric_key_prefix])
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 210, in compute_pet_metrics
centroids = self._compute_per_token_train_centroids(model)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 287, in _compute_per_token_train_centroids
mask_embeds.append(self.get_masks_embeds(model, batch))
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/trainers/trainer.py", line 267, in get_masks_embeds
hidden_states = model.roberta(input_ids=input_ids, attention_mask=attention_mask)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 868, in forward
encoder_outputs = self.encoder(
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 541, in forward
layer_outputs = layer_module(
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 468, in forward
layer_output = apply_chunking_to_forward(
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/transformers/modeling_utils.py", line 2196, in apply_chunking_to_forward
return forward_fn(*input_tensors)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 481, in feed_forward_chunk
layer_output = self.output(intermediate_output, attention_output)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/Documents/Deep Learning Project/perfect/fewshot/third_party/models/roberta/modeling_roberta.py", line 393, in forward
hidden_states = self.dense(hidden_states)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/timdadum/anaconda3/envs/perfect/lib/python3.8/site-packages/torch/nn/modules/linear.py", line 114, in forward
return F.linear(input, self.weight, self.bias)
KeyboardInterrupt
Which seems to suggest that somehow PyTorch can't comprehend the data processing done in Arrow. I'm not sure how to fix this, so any help would be welcome!