如何解决即使模型在训练和验证数据中显示出良好的准确性,也可能导致错误的预测
我使用了kaggle中的皮肤癌分类比赛数据。有4个标签,整个数据不平衡。我在10倍交叉验证拆分上运行resnet 18模型,以训练数据,每次折叠大约2个纪元。该代码已附在下面。 基本上,该模型给出了98.2%的准确度,其中火车数据的损失值为0.07,在验证数据中给出了98.1%的准确度,损失为0.06。因此,这看起来还不错。 但是问题是... prediction.py(下面附有代码)。当我尝试进行预测时,模型始终将结果设为[0]。即使是火车图像数据。
我的代码有问题吗?
预期结果: 如果图像是输入,则输出应为0、1,2或3
model.py(进行培训的地方)
import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import os
import cv2
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import albumentations as A
from torch.utils.data import TensorDataset,DataLoader,Dataset
from torchvision import models
from collections import defaultdict
from torch.utils.data.sampler import RandomSampler
import torch.optim as optim
from torch.optim import lr_scheduler
from sklearn import model_selection
from tqdm import tqdm
import gc
# generate data from csv file
class Build_dataset(Dataset):
def __init__(self,csv,split,mode,transform=None):
self.csv = csv.reset_index(drop=True)
self.split = split
self.mode = mode
self.transform = transform
def __len__(self):
return self.csv.shape[0]
def __getitem__(self,index):
row = self.csv.iloc[index]
image = cv2.imread(row.filepath)
image = cv2.cvtColor(image,cv2.COLOR_RGB2BGR)
if self.transform is not None:
res = self.transform(image=image)
image = res['image'].astype(np.float32)
else:
image = image.astype(np.float32)
image = image.transpose(2,1)
data = torch.tensor(image).float()
if self.mode == 'test':
return data
else:
return data,torch.tensor(self.csv.iloc[index].target).long()
# training data
def train_epoch(model,loader,optimizer,loss_fn,device,scheduler,n_examples):
model = model.train()
losses = []
correct_predictions = 0
for inputs,labels in tqdm(loader):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
_,preds = torch.max(outputs,dim=1)
loss = loss_fn(outputs,labels)
correct_predictions += torch.sum(preds == labels)
losses.append(loss.item())
loss.backward()
optimizer.step()
optimizer.zero_grad()
# here you delete inputs and labels and then use gc.collect
del inputs,labels
gc.collect()
return correct_predictions.double() / n_examples,np.mean(losses)
# validation data
def val_epoch(model,n_examples):
model = model.eval()
losses = []
correct_predictions = 0
with torch.no_grad():
for inputs,labels in tqdm(loader):
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
_,dim=1)
loss = loss_fn(outputs,labels)
correct_predictions += torch.sum(preds == labels)
losses.append(loss.item())
# here you delete inputs and labels and then use gc.collect
del inputs,labels
gc.collect()
return correct_predictions.double() / n_examples,np.mean(losses)
def train(fold,model,num_epochs):
df_train = df[df.kfold != fold].reset_index(drop=True)
df_valid = df[df.kfold == fold].reset_index(drop=True)
# generate data
dataset_train = Build_dataset(df_train,'train',transform=transforms_train)
dataset_valid = Build_dataset(df_valid,'val',transform=transforms_val)
#load data
train_loader = DataLoader(dataset_train,batch_size = 64,sampler=RandomSampler(dataset_train),num_workers=4)
valid_loader = DataLoader(dataset_valid,batch_size = 32,shuffle = True,num_workers= 4 )
dataset_train_size = len(dataset_train)
dataset_valid_size = len(dataset_valid)
optimizer = optim.Adam(model.parameters(),lr = 1e-4)
model = model.to(device)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,patience = 3,threshold = 0.001,mode =
'max')
loss_fn = nn.CrossEntropyLoss().to(device)
history = defaultdict(list)
best_accuracy = 0.0
for epoch in range(num_epochs):
print(f'Epoch {epoch+1} / {num_epochs}')
print ('-'*30)
train_acc,train_loss = train_epoch(model,train_loader,dataset_train_size)
print(f'Train loss {train_loss} accuracy {train_acc}')
valid_acc,valid_loss = val_epoch(model,valid_loader,dataset_valid_size)
print(f'Val loss {valid_loss} accuracy {valid_acc}')
print()
history['train_acc'].append(train_acc)
history['train_loss'].append(train_loss)
history['val_acc'].append(valid_acc)
history['val_loss'].append(valid_loss)
if valid_acc > best_accuracy:
print('saving model')
torch.save(model.state_dict(),f'best_model_{fold}.bin')
best_accuracy = valid_acc
print(f'Best Accuracy: {best_accuracy}')
model.load_state_dict(torch.load(f'best_model_{fold}.bin'))
return model,history
if __name__ == '__main__':
#competition data -2020
data_dir = "../input/jpeg-melanoma-384x384"
#competition data - 2019
data_dir2 = "../input/jpeg-isic2019-384x384"
# device
device = torch.device("cuda")
# augmenting images
image_size = 384
transforms_train = A.Compose([
A.Transpose(p=0.5),A.VerticalFlip(p=0.5),A.HorizontalFlip(p=0.5),A.RandomBrightness(limit=0.2,p=0.75),A.RandomContrast(limit=0.2,A.OneOf([
A.MedianBlur(blur_limit=5),A.GaussianBlur(blur_limit=5),A.GaussNoise(var_limit=(5.0,30.0)),],p=0.7),A.OneOf([
A.Opticaldistortion(distort_limit=1.0),A.Griddistortion(num_steps=5,distort_limit=1.),A.ElasticTransform(alpha=3),A.CLAHE(clip_limit=4.0,A.HueSaturationValue(hue_shift_limit=10,sat_shift_limit=20,val_shift_limit=10,p=0.5),A.ShiftScaleRotate(shift_limit=0.1,scale_limit=0.1,rotate_limit=15,border_mode=0,p=0.85),A.Resize(image_size,image_size),A.Cutout(max_h_size=int(image_size * 0.375),max_w_size=int(image_size * 0.375),num_holes=1,A.normalize()
])
transforms_val = A.Compose([
A.Resize(image_size,A.normalize()
])
# create data
df_train = pd.read_csv(os.path.join(data_dir,"train.csv")) #/kaggle/input/siim-isic-melanoma-classification/train.csv
df_train.head()
df_train['is_ext'] = 0
df_train['filepath'] = df_train['image_name'].apply(lambda x: os.path.join(data_dir,f'{x}.jpg'))
# dataset from 2020 data
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('seborrheic keratosis','BKL'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lichenoid keratosis','BKL'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('solar lentigo','BKL'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lentigo NOS','BKL'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('cafe-au-lait macule','unkNown'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('atypical melanocytic proliferation','unkNown'))
# shuffle data
df = df_train.sample(frac=1).reset_index(drop=True)
# creating 8 different target values
new_target = {d: idx for idx,d in enumerate(sorted(df.diagnosis.unique()))}
df['target'] = df['diagnosis'].map(new_target)
mel_idx = new_target['melanoma']
# creating 10 fold cross validation data
df = df_train.sample(frac=1).reset_index(drop=True)
df['kfold'] = -1
y = df_train.target.values
kf = model_selection.StratifiedKFold(n_splits=10,shuffle=True)
idx = kf.get_n_splits(X=df,y=y)
print(idx)
for fold,(x,y) in enumerate(kf.split(X=df,y=y)):
df.loc[y,'kfold'] = fold
df = df[['filepath','diagnosis','target','is_ext','kfold']]
class_names = list(df['diagnosis'].unique())
# create model
def create_model(n_classes):
model = models.resnet18(pretrained=True)
n_features = model.fc.in_features
model.fc = nn.Linear(n_features,n_classes)
return model.to(device)
base_model = create_model(len(class_names)) # model ready
# run the model
for i in range(10):
#train
base_model,history = train(i,base_model,num_epochs = 2) # train data
prediction.py
from torchvision import models
import torch
import torch.nn as nn
import albumentations as A
import cv2
import os
import numpy as np
device = torch.device("cuda")
MODEL = None
MODEL_PATH = "../input/prediction/best_model_4.bin"
def create_model(n_classes):
model = models.resnet18(pretrained=True)
n_features = model.fc.in_features
model.fc = nn.Linear(n_features,n_classes)
return model.to(device)
# generate the data to tensor with transform application
# converting the image to tensor by using the transforms function
class get_image:
def __init__(self,image_path,targets,transform = None):
self.image_path = image_path
self.targets = targets
self.transform = transform
def __len__(self):
return len(self.image_path)
def __getitem__(self,item):
targets = self.targets[item]
resize = 384
image = cv2.imread(self.image_path[item])
image = cv2.resize(image,(resize,resize))
image = cv2.cvtColor(image,cv2.COLOR_RGB2BGR)
if self.transform is not None:
res = self.transform(image = image)
image = res['image'].astype(np.float32)
image = image.transpose(2,1)
data = torch.tensor(image).float()
targets = torch.tensor(targets)
return data,targets
# load the data by using torch data
# predict values
# predict function
def predict(image_path,model_path):
image_size = 384
transforms_val = A.Compose([
A.Resize(image_size,A.normalize()
])
test_images = [image_path]
test_targets = [0]
test_data = get_image(
image_path = test_images,targets = test_targets,transform=transforms_val)
# loading the data
test_DataLoader = torch.utils.data.DataLoader(test_data,batch_size=1,shuffle = False,num_workers=0)
model = create_model(n_classes = 4)
model.load_state_dict(torch.load(model_path))
model.to(device)
model.eval()
prediction = []
with torch.no_grad():
for test_data,test_target in test_DataLoader:
test_data = test_data.to(device)
test_target = test_target.to(device)
outputs = model(test_data)
_,preds = torch.max(outputs.cpu(),1)
#prediction.extend(preds)
prediction = np.vstack((preds)).ravel()
return prediction
def upload_predict():
image_file = "../input/whatever/ISIC_0075663.jpg"
if image_file:
pred = predict(image_file,MODEL,MODEL_PATH)
print(pred)
return pred
标签及其计数在此处给出
3 27126
2 5193
1 584
0 223
这里0被认为是恶性癌症,其他标签属于不同类型。
以下是数据链接:https://www.kaggle.com/cdeotte/jpeg-melanoma-384x384
解决方法
我认为您可能会回答您的问题!你说:
有4个标签,整个数据不平衡
假设标签0没有癌症,而1,2,3是患有不同类型皮肤癌的病例。如果您说预测类是不平衡的,那么我猜整个样本的98%为0,因此您的算法只是将每种情况都预测为0,这样98%的时间它就会正确。当您的算法进入测试集时,它将简单地预测一切都为0。
因此问题不出在您的代码上。您必须通过上采样少数类,下采样多数类,为数据分配权重/偏差或使用某种模型集合来平衡数据集,请参见https://elitedatascience.com/imbalanced-classes。查看信用卡欺诈检测教程,例如https://towardsdatascience.com/credit-card-fraud-detection-1b3b3b44109b。
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。