I can't reproduce my results each times. I try to set random seed and only use the one GPU. My results are different very much, the acc often differs from 1% to 5%, or even 10%. I think there is something wrong with the code, but I don't know. I'm not a professional programmer, so I looked up some posts, but they all said that you just need to set the appropriate random seed. I think I fixed all the seeds on the official website, but the results still cannot be repeated. I don't know where to start to check this code, I hope you can help me. I am not my code:
import numpy as np
import pandas as pd
import sys, os
from random import shuffle
import torch
import torch.nn as nn
from models.gat import GATNet
from models.gat_gcn import GAT_GCN
from models.gcn import GCNNet
from models.ginconv import GINConvNet
from utils import *
import matplotlib.pyplot as plt
def set_random_seed(seed):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
os.environ['PYTHONHASHSEED'] = str(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
# training function at each epoch
def train(model, device, train_loader, optimizer, epoch):
print('Training on {} samples...'.format(len(train_loader.dataset)))
model.train()
total_train = torch.Tensor()
total_label = torch.Tensor()
train_losses = []
for batch_idx, data in enumerate(train_loader):
data = data.to(device)
optimizer.zero_grad()
output = model(data)
loss = loss_fn(output, data.y.view(-1,1).float()).to(device)
#loss = torch.mean(loss).float()
loss.backward()
optimizer.step()
train_losses.append(loss.item())
if batch_idx % LOG_INTERVAL == 0:
print('Train epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(epoch,
batch_idx * len(data.x),
len(train_loader.dataset),
100. * batch_idx / len(train_loader),
loss.item()))
total_train = torch.cat((total_train, output.cpu()), 0)
total_label = torch.cat((total_label, data.y.view(-1, 1).cpu()), 0)
G_train = total_label.detach().numpy().flatten()
P_train = total_train.detach().numpy().flatten()
ret = [auc(G_train,P_train),pre(G_train,P_train),recall(G_train,P_train),f1(G_train,P_train),acc(G_train,P_train),mcc(G_train,P_train),spe(G_train,P_train)]
print('train_auc',ret[0])
print('train_pre',ret[1])
print('train_recall',ret[2])
print('train_f1',ret[3])
print('train_acc',ret[4])
print('train_mcc',ret[5])
print('train_spe',ret[6])
print('train_loss',np.average(train_losses))
return G_train, P_train, np.average(train_losses)
def predicting(model, device, loader):
model.eval()
total_preds = torch.Tensor()
total_labels = torch.Tensor()
print('Make prediction for {} samples...'.format(len(loader.dataset)))
losses = []
with torch.no_grad():
for data in loader:
data = data.to(device)
output = model(data)
loss = loss_fn(output, data.y.view(-1,1).float())
#loss = torch.mean(loss).float().to(device)
losses.append(loss.item())
total_preds = torch.cat((total_preds, output.cpu()), 0)
total_labels = torch.cat((total_labels, data.y.view(-1, 1).cpu()), 0)
return total_labels.numpy().flatten(),total_preds.numpy().flatten(),np.average(losses)
datasets = [['cyp'][int(sys.argv[1])]]
modeling = [GINConvNet, GATNet, GAT_GCN, GCNNet][int(sys.argv[2])]
model_st = modeling.__name__
cuda_name = "cuda:0"
if len(sys.argv)>3:
cuda_name = ["cuda:0","cuda:1"][int(sys.argv[3])]
print('cuda_name:', cuda_name)
TRAIN_BATCH_SIZE = 512
TEST_BATCH_SIZE = 512
LR = 0.0005
LOG_INTERVAL = 20
NUM_EPOCHS = 1000
print('Learning rate: ', LR)
print('Epochs: ', NUM_EPOCHS)
#
set_random_seed(8)
# Main program: iterate over different datasets
for dataset in datasets:
print('\nrunning on ', model_st + '_' + dataset )
processed_data_file_train = 'data/processed/' + dataset + '_train.pt'
processed_data_file_test = 'data/processed/' + dataset + '_test.pt'
processed_data_file_valid = 'data/processed/' + dataset + '_valid.pt'
if ((not os.path.isfile(processed_data_file_train)) or (not os.path.isfile(processed_data_file_test))):
print('please run create_data.py to prepare data in pytorch format!')
else:
train_data = TestbedDataset(root='data', dataset=dataset+'_train')
test_data = TestbedDataset(root='data', dataset=dataset+'_test')
valid_data = TestbedDataset(root='data', dataset=dataset+'_valid')
train_set = pd.read_csv('try/cyp_train.csv')
#train_size = int(0.8 * len(train_data))
#valid_size = len(train_data) - train_size
#train_data, valid_data = torch.utils.data.random_split(train_data, [train_size, valid_size])
# make data PyTorch mini-batch processing ready
train_loader = DataLoader(train_data, batch_size=TRAIN_BATCH_SIZE, shuffle=True)
valid_loader = DataLoader(valid_data, batch_size=TEST_BATCH_SIZE, shuffle=False)
test_loader = DataLoader(test_data, batch_size=TEST_BATCH_SIZE, shuffle=False)
#pos_weight_np = pos_weight(train_set)
# training the model
device = torch.device(cuda_name if torch.cuda.is_available() else "cpu")
model = modeling().to(device)
loss_fn = nn.BCELoss()
#loss_fn = torch.nn.BCEWithLogitsLoss(reduction='none', pos_weight=pos_weight_np.to(device))
optimizer = torch.optim.Adam(model.parameters(), lr=LR)
best_loss = 100
best_test_auc = 1000
best_test_ci = 0
best_epoch = -1
patience = 20
early_stopping = EarlyStopping(patience=patience, verbose=True)
model_file_name = 'model_' + model_st + '_' + dataset + '.model'
result_file_name = 'result_' + model_st + '_' + dataset + '.csv'
train_losses=[]
train_accs=[]
valid_losses=[]
valid_accs=[]
for epoch in range(NUM_EPOCHS):
G_T,P_T,train_loss = train(model, device, train_loader, optimizer, epoch+1)
print('predicting for valid data')
G,P,loss_valid = predicting(model, device, valid_loader)
loss_valid_value = loss_valid
print('valid_loss',loss_valid)
print('valid_auc',auc(G,P))
print('valid_pre',pre(G,P))
print('valid_recall',recall(G,P))
print('valid_f1',f1(G,P))
print('valid_acc',acc(G,P))
print('valid_mcc',mcc(G,P))
print('valid_spe',spe(G,P))
train_losses.append(np.array(train_loss))
valid_losses.append(loss_valid)
train_accs.append(acc(G_T,P_T))
valid_accs.append(acc(G,P))
b = pd.DataFrame({'value':G,'prediction':P})
names = 'model_'+'value_validation'+'.csv'
b.to_csv(names,sep=',')
early_stopping(loss_valid, model)
if early_stopping.early_stop:
print("Early stopping")
torch.save(model.state_dict(), model_file_name)
print('predicting for test data')
G,P,loss_test = predicting(model, device, test_loader)
ret = [auc(G,P),pre(G,P),recall(G,P),f1(G,P),acc(G,P),mcc(G,P),spe(G,P)]
with open(result_file_name,'w') as f:
f.write(','.join(map(str,ret)))
best_test_loss = loss_test
print('auc improved at epoch ',best_epoch,'auc',ret[0],'pre',ret[1],'recall',ret[2],'f1',ret[3],'acc',ret[4],'mcc',ret[5],'spe',ret[6])
a = pd.DataFrame({'value':G,'prediction':P})
name = 'model_'+'value_test'+'.csv'
a.to_csv(name,sep=',')
break
else:
print('no early stopping')
df = pd.DataFrame({'train_loss':train_losses,'valid_loss':valid_losses,'train_accs':train_accs,'valid_accs':valid_accs})
names = 'model_'+'loss_acc'+'.csv'
df.to_csv(names,sep=',')
