训练期间 Huggingface 错误：AttributeError: 'str' object has no attribute 'size'

如何解决训练期间 Huggingface 错误：AttributeError: 'str' object has no attribute 'size'

在尝试使用 Pytorch Lightning 微调 Huggingface GPT2LMHeadModel 模型以进行休闲语言建模（给定单词序列，预测下一个单词）时，我在训练过程中遇到错误：

AttributeError: 'str' 对象没有属性 'size'

我们的训练代码出了什么问题？这是由于在 Pytorch DataCollatorForLanguageModeling 中错误使用了 DataLoader 吗？

可重现的示例：

import os
from pathlib import Path
import torch
import pytorch_lightning as pl
from transformers import (
    GPT2Config,GPT2LMHeadModel,GPT2Tokenizer,DataCollatorForLanguageModeling,)
from transformers.optimization import AdamW
from tokenizers import ByteLevelBPETokenizer
from torch.utils.data import (
    DataLoader,Dataset,)

TOKENIZER_DIRPATH = os.path.join("..","data")


def tokenize_data():
    tokenizer = ByteLevelBPETokenizer()
    tokenizer.train(
        files=os.path.join(TOKENIZER_DIRPATH,"words.txt"),vocab_size=50000,min_frequency=2,special_tokens=["<s>","</s>","<unk>","<mask>","<pad>",],)
    tokenizer.save_model("../data")


class MyDataset(Dataset):
    def __init__(self):
        tokenizer = GPT2Tokenizer(
            os.path.join(TOKENIZER_DIRPATH,"vocab.json"),os.path.join(TOKENIZER_DIRPATH,"merges.txt"),)

        src_file = Path(os.path.join(TOKENIZER_DIRPATH,"words.txt"))
        lines = src_file.read_text(encoding="utf-8").splitlines()
        self.examples = [tokenizer.encode(line) for line in lines]

    def __len__(self):
        return len(self.examples)

    def __getitem__(self,i):
        return torch.tensor(self.examples[i])


class MyDataModule(pl.LightningDataModule):
    def __init__(self):
        super().__init__()
        self.tokenizer = GPT2Tokenizer(
            os.path.join(TOKENIZER_DIRPATH,)

    def setup(self,stage):
        self.train_dataset = MyDataset()

    def train_dataloader(self):
        data_collator = DataCollatorForLanguageModeling(
            tokenizer=self.tokenizer,mlm=False
        )
        train_dataloader = DataLoader(self.train_dataset,collate_fn=data_collator)
        return train_dataloader


class MyModel(pl.LightningModule):
    def __init__(self,learning_rate,adam_beta1,adam_beta2,adam_epsilon):
        super().__init__()
        self.save_hyperparameters()
        config = GPT2Config()
        self.model = GPT2LMHeadModel(config)

    def forward(self,x):
        return self.model(x).logits

    def training_step(self,batch,batch_idx):
        input_ids,labels = batch
        loss = self.model(input_ids,labels=labels).loss
        self.log("train_loss",loss,on_epoch=True)
        return loss

    def configure_optimizers(self):
        optimizer = AdamW(
            self.parameters(),self.hparams.learning_rate,betas=(self.hparams.adam_beta1,self.hparams.adam_beta2),eps=self.hparams.adam_epsilon,)
        return optimizer


tokenize_data()
dm = MyDataModule()
model = MyModel(
    learning_rate=5e-5,adam_beta1=0.9,adam_beta2=0.999,adam_epsilon=1e-8,)

trainer = pl.Trainer()
trainer.fit(model,dm)

错误追溯：

Epoch 0:   0%|                                                                                                                                                                                                                                             | 0/9 [00:00<?,?it/s]
Traceback (most recent call last):
  File "test_gpt.py",line 102,in <module>
    trainer.fit(model,dm)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py",line 499,in fit
    self.dispatch()
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py",line 546,in dispatch
    self.accelerator.start_training(self)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py",line 73,in start_training
    self.training_type_plugin.start_training(trainer)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py",line 114,in start_training
    self._results = trainer.run_train()
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py",line 637,in run_train
    self.train_loop.run_training_epoch()
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py",line 493,in run_training_epoch
    batch_output = self.run_training_batch(batch,batch_idx,dataloader_idx)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py",line 655,in run_training_batch
    self.optimizer_step(optimizer,opt_idx,train_step_and_backward_closure)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py",line 426,in optimizer_step
    model_ref.optimizer_step(
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/core/lightning.py",line 1387,in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py",line 214,in step
    self.__optimizer_step(*args,closure=closure,profiler_name=profiler_name,**kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py",line 134,in __optimizer_step
    trainer.accelerator.optimizer_step(optimizer,self._optimizer_idx,lambda_closure=closure,**kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py",line 277,in optimizer_step
    self.run_optimizer_step(optimizer,lambda_closure,line 282,in run_optimizer_step
    self.training_type_plugin.optimizer_step(optimizer,lambda_closure=lambda_closure,**kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py",line 163,in optimizer_step
    optimizer.step(closure=lambda_closure,**kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/transformers/optimization.py",line 318,in step
    loss = closure()
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py",line 649,in train_step_and_backward_closure
    result = self.training_step_and_backward(
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py",line 743,in training_step_and_backward
    result = self.training_step(split_batch,hiddens)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py",line 293,in training_step
    training_step_output = self.trainer.accelerator.training_step(args)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py",line 156,in training_step
    return self.training_type_plugin.training_step(*args)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py",line 125,in training_step
    return self.lightning_module.training_step(*args,**kwargs)
  File "test_gpt.py",line 81,in training_step
    loss = self.model(input_ids,labels=labels).loss
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/torch/nn/modules/module.py",line 727,in _call_impl
    result = self.forward(*input,**kwargs)
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/transformers/models/gpt2/modeling_gpt2.py",line 904,in forward
    transformer_outputs = self.transformer(
  File "/opt/anaconda3/envs/test_huggingface/lib/python3.8/site-packages/torch/nn/modules/module.py",line 633,in forward
    input_shape = input_ids.size()
AttributeError: 'str' object has no attribute 'size'

Conda 包：

pytorch                   1.7.0           py3.8_cuda10.2.89_cudnn7.6.5_0    pytorch
pytorch-lightning         1.2.5              pyhd8ed1ab_0    conda-forge
tokenizers                0.10.1                   pypi_0    pypi
transformers              4.4.2                    pypi_0    pypi

解决方法

这里transformer最新版本可能会出现这个

pip install transformers==2.11.0

就我而言，这是可行的！！然后重启你的内核