from torch.utils.data import DataLoader
from transformers import PreTrainedModel

def convert_list_dict_to_dict_list(list_dict: dict[str, list]) -> list[dict[str, list]]:
    """ミニバッチのデータを事例単位のlistに変換"""
    dict_list = []
    #dictのキーのリストを取得する
    keys = list(list_dict.keys())
    for idx in range(len(list_dict[keys[0]])):  #各事例で処理する
        #dictの各キーからデータを取り出してlistに追加する
        dict_list.append({key: list_dict[key][idx] for key in keys})
    return dict_list

def run_generation(dataloader: DataLoader, model: PreTrainedModel) -> list[dict[str, Any]]:
    """見出しを生成"""
    generations = []
    for batch in tqdm(dataloader): #各ミニバッチを処理する
        #for k, v in batch.items():print(f"batch {batch}¥n key {k}¥n item{v}¥n¥n")
        batch = {k: v.to(model.device) for k, v in batch.items()}
        #見出しのトークンIDを生成する
        batch["generated_title_ids"] = model.generate(**batch)
        batch = {k: v.cpu().tolist() for k, v in batch.items()}
        #ミニバッチのデータ事例単位のlistに変換する
        generations += convert_list_dict_to_dict_list(batch)
    return generations

#テストセットに対して前処理を行う
test_dataset = dataset["test"].map(
    preprocess_data,
    fn_kwargs={"tokenizer": tokenizer},
    remove_columns=dataset["test"].column_names,
)

test_dataset = test_dataset.remove_columns(["labels"])

#ミニバッチの作成にDataLoaderを用いる
test_dataloader = DataLoader(
    test_dataset,
    batch_size=8,
    shuffle=False,
    collate_fn=data_collator,
)

#見出しを生成する
generations = run_generation(test_dataloader, model)

AttributeError                            Traceback (most recent call last)
[<ipython-input-19-2945c01206a3>](https://localhost:8080/#) in <cell line: 45>()
     43 )
     44 # 見出しを生成する
---> 45 generations = run_generation(test_dataloader, model)

1 frames
[<ipython-input-19-2945c01206a3>](https://localhost:8080/#) in <dictcomp>(.0)
     20     generations = []
     21     for batch in tqdm(dataloader):  # 各ミニバッチを処理する
---> 22         batch = {k: v.to(model.device) for k, v in batch.items()}
     23         # 見出しのトークンのIDを生成する
     24         batch["generated_title_ids"] = model.generate(**batch)

AttributeError: 'NoneType' object has no attribute 'to'

File /opt/conda/lib/python3.10/site-packages/datasets/builder.py:480, in DatasetBuilder._info(self)
    470 @abc.abstractmethod
    471 def _info(self) -> DatasetInfo:
    472     """Construct the DatasetInfo object. See `DatasetInfo` for details.
    473 
    474     Warning: This function is only called once and the result is cached for all
   (...)
    478         info: (DatasetInfo) The dataset information
    479     """
--> 480     raise NotImplementedError

NotImplementedError: 

from datasets import load_dataset
dataset = load_dataset("llm-book/livedoor-news-corpus")

File ~\.cache\huggingface\modules\datasets_modules\datasets\llm-book--livedoor-news-corpus\xxxx\livedoor-news-corpus.py:91 in (.0)
     89         continue
     90     with open(file_name, "r") as f:
---> 91         d = [line.strip() for line in f]
     92         data.append(
     93             {
     94                 "url": d[0],
   (...)
     99             }
    100         )
    102 if self.config.shuffle == True:

UnicodeDecodeError: 'cp932' codec can't decode byte 0x83 in position 96: illegal multibyte sequence