diff --git a/build_models/evaluation.py b/build_models/evaluation.py index efe864534043bfc96b2f19b01428a85fbff85cae..1e432308ad2652fa0f8c64350130de7fd8b2d493 100644 --- a/build_models/evaluation.py +++ b/build_models/evaluation.py @@ -4,8 +4,10 @@ from sklearn.metrics import confusion_matrix import seaborn as sns import matplotlib.pyplot as plt import itertools +import pandas as pd +import os -from parameters import SAVE_MISCLASSIFIED_PATH, SAVE_CONFUSIONM_PATH, SAVE_PATH_LEARNING_CURVES, MODEL_NAME, AXIS_CONVERTION +from parameters import SAVE_MISCLASSIFIED_PATH, SAVE_CONFUSIONM_PATH, SAVE_PATH_LEARNING_CURVES, MODEL_NAME, AXIS_CONVERTION, EPOCHS, SPECIALIZATION def get_learning_curves(train_losses, val_losses, train_accuracies, val_accuracies, savename = None): epochs = range(1, len(train_losses) + 1) @@ -124,11 +126,11 @@ def get_confusionM(y_true, y_pred_major, labels, test_set_len, averaging = False P = conf_matrix.sum(axis=1) # Calculate balanced accuracy - balanced_accuracy = sum(TP / P) / len(P) - + balanced_accuracy = sum(TP / P) / len(P) *100 + test_acc = 100-num_errors/test_set_len*100 print(f'Test errors {num_errors} (out of {test_set_len:.0f}) {num_errors/test_set_len*100:0.2f}%') - print(f'Test accuracy {100-num_errors/test_set_len*100:0.2f}%') - print(f'Balanced accuracy {balanced_accuracy*100:0.2f}%') + print(f'Test accuracy {test_acc:0.2f}%') + print(f'Balanced accuracy {balanced_accuracy:0.2f}%') plt.figure(figsize=(8, 6)) sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues', @@ -141,4 +143,38 @@ def get_confusionM(y_true, y_pred_major, labels, test_set_len, averaging = False plt.savefig(SAVE_CONFUSIONM_PATH + "averaging.png", dpi=300, bbox_inches='tight') else: plt.savefig(SAVE_CONFUSIONM_PATH, dpi=300, bbox_inches='tight') - plt.show() \ No newline at end of file + plt.show() + + return num_errors, test_acc, balanced_accuracy + +def save_results(best_epoch, valid_loss, valid_acc, test_acc, balanced_test_acc, num_err, num_all_test, add_spec = "", file_name = "results.xlsx"): + attributes = ["Model name", "Specialization", "Best epoch", "All epoch" ,"Validation loss", "Validation accuracy", + "Test accuracy", "Balanced test accuracy", "Num of errors", "Errors out of"] + new_row = { + "Model name": MODEL_NAME, + "Specialization": SPECIALIZATION + add_spec, + "Best epoch": best_epoch, + "All epoch": EPOCHS, # Placeholder if you don't have this value yet + "Validation loss": valid_loss, + "Validation accuracy": valid_acc, + "Test accuracy": test_acc, + "Balanced test accuracy": balanced_test_acc, + "Num of errors": num_err, # Placeholder + "Errors out of": num_all_test, # Placeholder + } + + # Check if the file exists + if os.path.exists(file_name): + # Read the existing table + existing_df = pd.read_excel(file_name) + else: + # Create a new table if it doesn't exist + existing_df = pd.DataFrame(columns=attributes) + + # Add the new row to the DataFrame + existing_df = pd.concat([existing_df, pd.DataFrame([new_row])], ignore_index=True) + + # Save the updated DataFrame to the Excel file + existing_df.to_excel(file_name, index=False) + + print(f"Table updated and saved to {file_name}") \ No newline at end of file diff --git a/build_models/main.py b/build_models/main.py index 9a479bddbe85361085859e43a65e0002868db948..90577ac2cdc79b8e9f74880cfa859e1a9044093d 100644 --- a/build_models/main.py +++ b/build_models/main.py @@ -14,14 +14,14 @@ from torch.utils.data import DataLoader import torch.nn as nn import torch.optim as optim -from parameters import (PIPELINE, TRAIN_FOLDER, VAL_FOLDER, TEST_FOLDER, DEVICE, BATCH_SIZE, WORKERS, MODEL_NAME, EPOCHS, +from parameters import (PIPELINE, TRAIN_FOLDER, VAL_FOLDER, TEST_FOLDER, INVIDUALS, DEVICE, BATCH_SIZE, WORKERS, MODEL_NAME, BEST_EPOCH, EPOCHS, CHECKPOINTS_FOLDER, SAVE_PREDICTION_PATH, LOG_PATH) from own_dataloader import (different_anatomical_plane_in_rgb, same_anatomical_plane_in_rgb, different_anatomical_plane_ensemble, ImageTripletDataset_to_one, same_anatomical_planes_in_RGB_ensemble, ImageTripletDataset_to_three, visualize_RGB_image_from_dataloader, get_data_distribution,visualize_ENSEMBLE_image_from_dataloader, RandomResample) from train_functions import train_with_scaler from test_functions import predict, averaging_prediction, predict_ENSEMBLE_with_confidence -from evaluation import get_learning_curves, averaged_misclassified_data, get_misclassified_data, get_confusionM +from evaluation import get_learning_curves, averaged_misclassified_data, get_misclassified_data, get_confusionM, save_results # FIX THE SEEDS random.seed(42) @@ -198,6 +198,7 @@ elif PIPELINE == "different_anatomical_plane_ensemble" or PIPELINE == "ensemble_ optimizer_Adam2 = optim.Adam(model2.parameters(), 1e-3) scaler2 = torch.amp.GradScaler('cuda') checkpoints_folder2 = CHECKPOINTS_FOLDER + '_ensemble_2_checkpoints' + if not os.path.exists(checkpoints_folder2): os.mkdir(checkpoints_folder2) train_losses2, val_losses2, train_accuracies2, val_accuracies2, BEST_EPOCH2 = train_with_scaler(model2, train_coronial_loader, val_coronial_loader, @@ -210,11 +211,13 @@ elif PIPELINE == "different_anatomical_plane_ensemble" or PIPELINE == "ensemble_ optimizer_Adam3 = optim.Adam(model3.parameters(), 1e-3) scaler3 = torch.amp.GradScaler('cuda') checkpoints_folder3 = CHECKPOINTS_FOLDER + 'ensemble_3_checkpoints' + if not os.path.exists(checkpoints_folder3): os.mkdir(checkpoints_folder3) train_losses3, val_losses3, train_accuracies3, val_accuracies3, BEST_EPOCH3 = train_with_scaler(model3, train_sagittal_loader, val_sagittal_loader, optimizer_Adam3, criterion_balanced3, EPOCHS, scaler3, DEVICE, checkpoints_foler=checkpoints_folder3) + get_learning_curves(train_losses1, val_losses1, train_accuracies1, val_accuracies1, "_1_model") get_learning_curves(train_losses2, val_losses2, train_accuracies2, val_accuracies2, "_2_model") get_learning_curves(train_losses3, val_losses3, train_accuracies3, val_accuracies3, "_3_model") @@ -241,32 +244,39 @@ elif PIPELINE == "different_anatomical_plane_ensemble" or PIPELINE == "ensemble_ model3 = torch.load(checkpoints_folder3+f'/avp_{BEST_EPOCH3:03d}.pkl') y_pred_majority, y_true, confidence_scores_majority, gt_labels_conf = predict_ENSEMBLE_with_confidence(model1, model2, model3, test_axial_loader, test_coronial_loader, test_sagittal_loader, DEVICE) +#val_losses = None +#val_accuracies = None #EVALUATION if PIPELINE == "different_anatomical_plane_in_rgb": # different and same anatomical planes in RGB y_pred_majority = y_pred.argmax(axis=1) # find the argmax of each of the predictions confidence_scores_majority = y_pred.max(axis = 1) #get the confidence score y_true = [label for _, label in test_set.samples] # get the true labels and convert to numpy - get_confusionM(y_true, y_pred_majority, test_labels, len(test_set)) - get_misclassified_data(y_pred_majority, y_true, confidence_scores_majority, gt_labels_conf, test_image_paths, test_labels) - -elif PIPELINE == "different_anatomical_plane_ensemble" or PIPELINE == "ensemble_same_anatomical_planes_in_RGB": - # different and same anatomical plane ensemble - get_confusionM(y_true, y_pred_majority, test_labels, len(test_datasetAxial)) + num_errors, test_acc, balanced_accuracy = get_confusionM(y_true, y_pred_majority, test_labels, len(test_set)) get_misclassified_data(y_pred_majority, y_true, confidence_scores_majority, gt_labels_conf, test_image_paths, test_labels) + save_results(BEST_EPOCH, val_losses, val_accuracies, test_acc, balanced_accuracy, num_errors, len(test_set)) elif PIPELINE == "basic" or PIPELINE == "same_anatomical_plane_in_rgb": y_pred_majority = y_pred.argmax(axis=1) # find the argmax of each of the predictions confidence_scores_majority = y_pred.max(axis = 1) #get the confidence score y_true = [label for _, label in test_set.samples] # get the true labels and convert to numpy - get_confusionM(y_true, y_pred_majority, test_labels, len(test_set)) + num_errors, test_acc, balanced_accuracy = get_confusionM(y_true, y_pred_majority, test_labels, len(test_set)) get_misclassified_data(y_pred_majority, y_true, confidence_scores_majority, gt_labels_conf, test_image_paths, test_labels) + save_results(BEST_EPOCH, val_losses, val_accuracies, test_acc, balanced_accuracy, num_errors, len(test_set)) # averaging evaluation -> simaknal meg a same anatomical planes in RGB grouped_predictions, ground_truth_dict, final_predictions, grouped_gt_predictions = averaging_prediction(y_pred, test_set, test_image_paths) y_true, y_pred_major = averaged_misclassified_data(final_predictions, ground_truth_dict, grouped_predictions, grouped_gt_predictions, test_labels) - get_confusionM(y_true, y_pred_major, test_labels, len(test_set)/3, averaging=True) - + AVG_num_errors, AVG_test_acc, AVG_balanced_accuracy = get_confusionM(y_true, y_pred_major, test_labels, INVIDUALS, averaging=True) + save_results(BEST_EPOCH, val_losses, val_accuracies, AVG_test_acc, AVG_balanced_accuracy, AVG_num_errors, INVIDUALS) + +elif PIPELINE == "different_anatomical_plane_ensemble" or PIPELINE == "ensemble_same_anatomical_planes_in_RGB": + # different and same anatomical plane ensemble + num_errors, test_acc, balanced_accuracy = get_confusionM(y_true, y_pred_majority, test_labels, len(test_datasetAxial)) + get_misclassified_data(y_pred_majority, y_true, confidence_scores_majority, gt_labels_conf, test_image_paths, test_labels) + save_results(BEST_EPOCH1, val_losses1, val_accuracies1, test_acc, balanced_accuracy, num_errors, len(test_datasetAxial)) + save_results(BEST_EPOCH2, val_losses2, val_accuracies2, test_acc, balanced_accuracy, num_errors, len(test_datasetCoronial)) + save_results(BEST_EPOCH3, val_losses3, val_accuracies3, test_acc, balanced_accuracy, num_errors, len(test_datasetSagittal)) sys.stdout = sys.__stdout__ # Reset stdout back to normal log_file.close() diff --git a/build_models/parameters.py b/build_models/parameters.py index d01d6a6ccf49b2e19d906cb7eb019297723fdb24..ae4af34f6e12cfd6eb8d7fca648d571f93ffbf73 100644 --- a/build_models/parameters.py +++ b/build_models/parameters.py @@ -48,6 +48,7 @@ DEVICE = 'cuda' # 'cuda' or 'cpu' TRAIN_FOLDER = DATA_PATH + '/train' VAL_FOLDER = DATA_PATH + '/val' TEST_FOLDER = DATA_PATH + '/test' +INVIDUALS = 1940 MODEL_NAME = 'resnet18' @@ -57,7 +58,7 @@ EPOCHS = 60 SAVE_FOLDER_MAIN = '/net/travail/mvajay/TRDP/' -#BEST_EPOCH = 23 +BEST_EPOCH = 23 CHECKPOINTS_FOLDER = SAVE_FOLDER_MAIN+'checkpoints/transfer_checkpoints_'+ MODEL_NAME + SPECIALIZATION SAVE_PREDICTION_PATH = '/net/cremi/mvajay/TRDP/predictions/' + MODEL_NAME + SPECIALIZATION + "_b_epoch_" diff --git a/build_models/run_combinations.py b/build_models/run_combinations.py index d313adeaa2a4d4a7851735af62ced0293c0b0ccb..7bc5cac4330a59995f608e3c61ce8455c1d91714 100644 --- a/build_models/run_combinations.py +++ b/build_models/run_combinations.py @@ -35,7 +35,8 @@ DEVICE = 'cuda' # 'cuda' or 'cpu' TRAIN_FOLDER = DATA_PATH + '/train' VAL_FOLDER = DATA_PATH + '/val' TEST_FOLDER = DATA_PATH + '/test' - +INVIDUALS = 1940 + MODEL_NAME = 'resnet18' BATCH_SIZE = 64 @@ -52,7 +53,7 @@ SAVE_INPUT_VISUALIZATION_PATH = SAVE_FOLDER_MAIN+'input_visualization/' + MODEL_ SAVE_PATH_LEARNING_CURVES = SAVE_FOLDER_MAIN + "learning_curves/" + MODEL_NAME + SPECIALIZATION LOG_PATH = SAVE_FOLDER_MAIN+'logs/'+ MODEL_NAME + SPECIALIZATION + ".log" - + AXIS_CONVERTION = {'0': "axial",'1': "coronial",'2': "sagittal"} """)