resample augmentation training and evaluation

augmentation_robustness_exploration/transform_evaluation_results_resnet18_flips_90_resample.csv

+use_random_hflip,use_random_vflip,random_rotation,random_contrast,resample,balanced_accuracy
+False,False,0,0,1,0.8854370606457157
+False,False,0,0,2,0.9636540639212855
+False,False,0,0,4,0.9571377296205392
+False,False,10,0,1,0.9670414745075497
+False,False,10,0,2,0.9694108030154761
+False,False,10,0,4,0.9644360164870679
+False,False,20,0,1,0.9680933173165492
+False,False,20,0,2,0.9556037433066721
+False,False,20,0,4,0.720484345531819
+False,False,30,0,1,0.9706325988526704
+False,False,30,0,2,0.926098680276943
+False,False,30,0,4,0.9702318911660334
+False,False,40,0,1,0.9725376938296153
+False,False,40,0,2,0.9712059320009596
+False,False,40,0,4,0.9708555276416792
+False,False,50,0,1,0.9720814191818915
+False,False,50,0,2,0.9688659675836355
+False,False,50,0,4,0.8963337434070568
+False,False,60,0,1,0.9679402076735762
+False,False,60,0,2,0.9676700121100187
+False,False,60,0,4,0.9705905677783973
+False,False,70,0,1,0.9727127124792869
+False,False,70,0,2,0.9674864772743285
+False,False,70,0,4,0.8739456869430536
+False,False,80,0,1,0.968232833924331
+False,False,80,0,2,0.9709196188085674
+False,False,80,0,4,0.8492338107456586
+False,False,90,0,1,0.9686320021479106
+False,False,90,0,2,0.9535043624406931
+False,False,90,0,4,0.9687437879850648
+False,False,100,0,1,0.9690221031245626
+False,False,100,0,2,0.9503350371232655
+False,False,100,0,4,0.7439363905408073
+False,False,110,0,1,0.9732230894339124
+False,False,110,0,2,0.9700337592607925
+False,False,110,0,4,0.9671116824348286
+False,False,120,0,1,0.9683307230735592
+False,False,120,0,2,0.9120535251013313
+False,False,120,0,4,0.9662137319911058
+False,False,130,0,1,0.9680709847040804
+False,False,130,0,2,0.9702564359400825
+False,False,130,0,4,0.9699305392729036
+False,False,140,0,1,0.9733476016690216
+False,False,140,0,2,0.9438535730922373
+False,False,140,0,4,0.8410845960054513
+False,False,150,0,1,0.9701722609884131
+False,False,150,0,2,0.9518747759339019
+False,False,150,0,4,0.7129328981385444
+False,False,160,0,1,0.9705094619551294
+False,False,160,0,2,0.9704246739441148
+False,False,160,0,4,0.35576619903890055
+False,False,170,0,1,0.9717071408800457
+False,False,170,0,2,0.9698583406488581
+False,False,170,0,4,0.8725409383409047
+False,True,0,0,1,0.8873194068640929
+False,True,0,0,2,0.9529557070378446
+False,True,0,0,4,0.7779667907572902
+False,True,10,0,1,0.9654262685870958
+False,True,10,0,2,0.9206905877117497
+False,True,10,0,4,0.8079172456363135
+False,True,20,0,1,0.9675485085069927
+False,True,20,0,2,0.9617546115361407
+False,True,20,0,4,0.9686223937184094
+False,True,30,0,1,0.9708527786977975
+False,True,30,0,2,0.9671053796451824
+False,True,30,0,4,0.9666649108434509
+False,True,40,0,1,0.9709127691503642
+False,True,40,0,2,0.9703662308555766
+False,True,40,0,4,0.9711993860251942
+False,True,50,0,1,0.9691511940912305
+False,True,50,0,2,0.9695663785888675
+False,True,50,0,4,0.968117037733317
+False,True,60,0,1,0.9709220391068465
+False,True,60,0,2,0.9658954127328548
+False,True,60,0,4,0.9627295868964001
+False,True,70,0,1,0.9726380984155076
+False,True,70,0,2,0.9712276242104428
+False,True,70,0,4,0.9721543957163713
+False,True,80,0,1,0.9735690884374544
+False,True,80,0,2,0.9701901053104065
+False,True,80,0,4,0.40786161057072284
+False,True,90,0,1,0.9691706179667082
+False,True,90,0,2,0.967194859683099
+False,True,90,0,4,0.8343815893560537
+False,True,100,0,1,0.9709809839282654
+False,True,100,0,2,0.9680639322175111
+False,True,100,0,4,0.6898193436179225
+False,True,110,0,1,0.9697345628603556
+False,True,110,0,2,0.9678008647853199
+False,True,110,0,4,0.958818673720892
+False,True,120,0,1,0.9699075639660831
+False,True,120,0,2,0.9051814521019453
+False,True,120,0,4,0.8006683581488209
+False,True,130,0,1,0.9703459529071888
+False,True,130,0,2,0.9681124804302781
+False,True,130,0,4,0.7671464044785947
+False,True,140,0,1,0.9686373265155824
+False,True,140,0,2,0.9457896241375671
+False,True,140,0,4,0.6420686236296476
+False,True,150,0,1,0.96843870548422
+False,True,150,0,2,0.9729745712156589
+False,True,150,0,4,0.970583080462605
+False,True,160,0,1,0.9735525706638962
+False,True,160,0,2,0.9724099614523238
+False,True,160,0,4,0.9679579585078171
+False,True,170,0,1,0.9687673107714735
+False,True,170,0,2,0.9713175550747147
+False,True,170,0,4,0.3947791302884985
+True,False,0,0,1,0.8840291855079485
+True,False,0,0,2,0.9418262474277351
+True,False,0,0,4,0.959299020531355
+True,False,10,0,1,0.9613510821632226
+True,False,10,0,2,0.958950602294595
+True,False,10,0,4,0.9453478370405628
+True,False,20,0,1,0.9663588544354722
+True,False,20,0,2,0.9132284406071774
+True,False,20,0,4,0.9131820947752508
+True,False,30,0,1,0.9674486218038715
+True,False,30,0,2,0.96688028470014
+True,False,30,0,4,0.966020367095613
+True,False,40,0,1,0.9657439311779028
+True,False,40,0,2,0.9669472390143127
+True,False,40,0,4,0.9679475031553179
+True,False,50,0,1,0.9708516235393319
+True,False,50,0,2,0.9648873139812774
+True,False,50,0,4,0.466692633500944
+True,False,60,0,1,0.9671795713125239
+True,False,60,0,2,0.9711845865091714
+True,False,60,0,4,0.9698955015764316
+True,False,70,0,1,0.9709079578875398
+True,False,70,0,2,0.8983614520904393
+True,False,70,0,4,0.8990836353629533
+True,False,80,0,1,0.9724089205990994
+True,False,80,0,2,0.9696844912750157
+True,False,80,0,4,0.9732184599910185
+True,False,90,0,1,0.9666291877907774
+True,False,90,0,2,0.9705176544644161
+True,False,90,0,4,0.8157221992174544
+True,False,100,0,1,0.9685251512182622
+True,False,100,0,2,0.9503460682622153
+True,False,100,0,4,0.9410608274105814
+True,False,110,0,1,0.967231654832203
+True,False,110,0,2,0.9697324676390877
+True,False,110,0,4,0.9530536360653361
+True,False,120,0,1,0.9733881836067526
+True,False,120,0,2,0.8927071812207407
+True,False,120,0,4,0.8804782189177037
+True,False,130,0,1,0.9682153957428034
+True,False,130,0,2,0.9045374078295679
+True,False,130,0,4,0.5049944948727912
+True,False,140,0,1,0.9693607956987228
+True,False,140,0,2,0.9685513157932235
+True,False,140,0,4,0.7215436893751067
+True,False,150,0,1,0.9739413394528054
+True,False,150,0,2,0.9628850471801113
+True,False,150,0,4,0.8892756931832313
+True,False,160,0,1,0.9728900684415817
+True,False,160,0,2,0.9693584426096172
+True,False,160,0,4,0.8098137455108809
+True,False,170,0,1,0.9710977538439524
+True,False,170,0,2,0.972274727977095
+True,False,170,0,4,0.9399125532791155
+True,True,0,0,1,0.8860381598516768
+True,True,0,0,2,0.9505829246047056
+True,True,0,0,4,0.2553616922496635
+True,True,10,0,1,0.9630790805945049
+True,True,10,0,2,0.9204667706363345
+True,True,10,0,4,0.9673581389112247
+True,True,20,0,1,0.9664993930690838
+True,True,20,0,2,0.9670849204963821
+True,True,20,0,4,0.9663349445990751
+True,True,30,0,1,0.9659838223971086
+True,True,30,0,2,0.9165998826986189
+True,True,30,0,4,0.4624251708607127
+True,True,40,0,1,0.9684906579080332
+True,True,40,0,2,0.9691249462293409
+True,True,40,0,4,0.7956516521738768
+True,True,50,0,1,0.9705140652722022
+True,True,50,0,2,0.9688444834217425
+True,True,50,0,4,0.9712334149900054
+True,True,60,0,1,0.9694788892562043
+True,True,60,0,2,0.9667787009770562
+True,True,60,0,4,0.9669778848825951
+True,True,70,0,1,0.9716106617104526
+True,True,70,0,2,0.9707352215873988
+True,True,70,0,4,0.9685407665074284
+True,True,80,0,1,0.9707693752067899
+True,True,80,0,2,0.9571139682180141
+True,True,80,0,4,0.7789004240317182
+True,True,90,0,1,0.9680773892643446
+True,True,90,0,2,0.9305860005876875
+True,True,90,0,4,0.9699958238555914
+True,True,100,0,1,0.9712690125521745
+True,True,100,0,2,0.9679473940438407
+True,True,100,0,4,0.9698676937734574
+True,True,110,0,1,0.9666012497618082
+True,True,110,0,2,0.971220518487792
+True,True,110,0,4,0.896894179493021
+True,True,120,0,1,0.9696342790209904
+True,True,120,0,2,0.9698025582102133
+True,True,120,0,4,0.9703521122488067
+True,True,130,0,1,0.9701415366620453
+True,True,130,0,2,0.9634308355870284
+True,True,130,0,4,0.8751676195281509
+True,True,140,0,1,0.9689168768543718
+True,True,140,0,2,0.9721418725541368
+True,True,140,0,4,0.8420782006631847
+True,True,150,0,1,0.9698486839436598
+True,True,150,0,2,0.9509325837903898
+True,True,150,0,4,0.9699399565981973
+True,True,160,0,1,0.9687291031179883
+True,True,160,0,2,0.9506131843395463
+True,True,160,0,4,0.7534493219040826
+True,True,170,0,1,0.9686626800953239
+True,True,170,0,2,0.9719511603396739
+True,True,170,0,4,0.9715887158466998

copy_best_models_to_local.py

@@ -22,9 +22,10 @@ model_folders = ['transfer_checkpoints_efficientnet_b0_adam_amp_criterion_balanc
                  'transfer_checkpoints_resnet18_flips',
                  'transfer_checkpoints_resnet18_flips_90',
                  'transfer_checkpoints_resnet18_flips_180',
-                 'transfer_checkpoints_resnet18_flips_360']
+                 'transfer_checkpoints_resnet18_flips_360',
+                 'transfer_checkpoints_resnet18_flips_90_resample']
 
-best_epochs = [26, 18, 25, 12, 22, 28, 4, 7, 21, 19, 3, 24, 22, 13, 22, 22]
+best_epochs = [26, 18, 25, 12, 22, 28, 4, 7, 21, 19, 3, 24, 22, 13, 22, 22, 16]
 
 for i in range(len(model_folders)):
     model_path = model_path_root + model_folders[i] + '/'

test_transform_exploration.py

@@ -107,7 +107,8 @@ model_path = '/net/cremi/bformanek/TRDP_II/local_models/'
 #model_name = 'transfer_checkpoints_resnet18_flips_avp_022.pkl' # augmented with flips
 #model_name = 'transfer_checkpoints_resnet18_flips_90_avp_013.pkl' # augmented with flips and rotation upto 90 degrees
 #model_name = 'transfer_checkpoints_resnet18_flips_180_avp_022.pkl' # augmented with flips and rotation upto 180 degrees
-model_name = 'transfer_checkpoints_resnet18_flips_360_avp_022.pkl' # augmented with flips and rotation upto 360 degrees
+#model_name = 'transfer_checkpoints_resnet18_flips_360_avp_022.pkl' # augmented with flips and rotation upto 360 degrees
+model_name = 'transfer_checkpoints_resnet18_flips_90_resample_avp_016.pkl' # augmented with flips and rotation upto 90 degrees plus resample upto half resolution
 model = torch.load(model_path + model_name, map_location=torch.device('cpu'), weights_only=False)
 model.to(device)
 
@@ -119,7 +120,7 @@ use_random_hflip = [False, True]
 use_random_vflip = [False, True]
 random_rotation = list(range(0, 180, 10))
 #random_contrast = [0, 0.1, 0.2, 0.3, 0.4, 0.5]
-random_contrast = [0, 0.2, 0.5]
+random_contrast = [0]
 use_resample = [1, 2, 4]
 
 BATCH_SIZE = 64
@@ -131,9 +132,9 @@ iterations = 0
 y_true_set = False # flag to calculate y_true only once
 for hflip, vflip, rotation, contrast, resample in product(use_random_hflip, use_random_vflip, random_rotation, random_contrast, use_resample):
     # skip iterations at the beginning if needed
-    # if iterations < 540:
-    #   iterations += 1
-    #   continue
+    if iterations < 180:
+      iterations += 1
+      continue
     
     transform = get_transform(hflip, vflip, rotation, contrast, resample)
 
@@ -173,11 +174,11 @@ for hflip, vflip, rotation, contrast, resample in product(use_random_hflip, use_
     if iterations % 36 == 0:
         # Save results to a DataFrame and export to CSV for easy analysis
         df_results = pd.DataFrame(results)
-        df_results.to_csv(f"transform_evaluation_results_resnet18_flips_360_{iterations}.csv", index=False) 
+        df_results.to_csv(f"transform_evaluation_results_resnet18_flips_90_resample_{iterations}.csv", index=False) 
         print(f"save at iteration {iterations}")       
 
     del test_set, test_loader, y_pred
     
 # Save results to a DataFrame and export to CSV for easy analysis
 df_results = pd.DataFrame(results)
-df_results.to_csv("transform_evaluation_results_resnet18_flips_360.csv", index=False)
\ No newline at end of file
+df_results.to_csv("transform_evaluation_results_resnet18_flips_90_resample.csv", index=False)
\ No newline at end of file

time_measure.py

@@ -128,10 +128,22 @@ test_image_folder = '/net/travail/bformanek/MRI_dataset/test'
 local_images_folder = '/net/cremi/bformanek/TRDP_II/local_images/'
 local_model_folder = '/net/cremi/bformanek/TRDP_II/local_models/'
 
+"""
+predefined_images_list = [
+    'T1w/4RTNI_5_S_5006_MR_Sag_IR-SPGR__br_raw_20140821092148809_194_S228381_I440459_2529.7502_0_093.png',
+    'T1w/4RTNI_5_S_5006_MR_Sag_IR-SPGR__br_raw_20140821092148809_194_S228381_I440459_2529.7502_0_093.png',
+    'T1w/ISYB_sub-0103_T1w_3526.7072_2_121.png',
+    'T1w/ADNI_035_S_0048_MR_MPR-R__GradWarp__B1_Correction__N3_Br_20070319172419398_S10258_I45187_617.3637_0_091.png'
+]"""
+
 for i in range(number_of_measurements):
     print(f"MEASUREMENT - {i}" )
     # get image to local folder
-    local_image_path = copy_image_to_local(test_image_folder, local_images_folder)
+    local_image_path = copy_image_to_local(test_image_folder, local_images_folder) # random image
+    
+    # local_image_path = os.path.join(local_images_folder, os.path.basename(predefined_images_list[i])) # predefined image
+    # shutil.copyfile(os.path.join(test_image_folder, predefined_images_list[i]), local_image_path) # predefined image
+    
     for model_name in os.listdir(local_model_folder) :
         print("MODEL: ")
         print(model_name)

time_measurement/final_local_time_measurement_log.txt

@@ -847,4 +847,62 @@ Prediction time for one image: 1.59 ms
 Compete inference time for one image: 24.52 ms
 GPU memory usage: 43.275390625 MB
 -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- 
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- 
+
+MEASUREMENT - 1
+MODEL: 
+transfer_checkpoints_resnet18_flips_90_resample_avp_016.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 6.15 ms
+Model loaded: 17.71 ms
+Prediction time for one image: 16.32 ms
+Compete inference time for one image: 40.21 ms
+DEVICE: cuda
+cuda is available
+Inference time masurement using cuda
+Image loaded: 1.28 ms
+Model loaded: 24.64 ms
+Prediction time for one image: 1.43 ms
+Compete inference time for one image: 27.49 ms
+GPU memory usage: 43.275390625 MB
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- 
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- 
+MEASUREMENT - 2
+MODEL: 
+transfer_checkpoints_resnet18_flips_90_resample_avp_016.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 13.00 ms
+Model loaded: 19.45 ms
+Prediction time for one image: 38.88 ms
+Compete inference time for one image: 71.35 ms
+DEVICE: cuda
+cuda is available
+Inference time masurement using cuda
+Image loaded: 1.22 ms
+Model loaded: 21.13 ms
+Prediction time for one image: 1.62 ms
+Compete inference time for one image: 24.10 ms
+GPU memory usage: 43.275390625 MB
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- 
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- 
+MEASUREMENT - 3
+MODEL: 
+transfer_checkpoints_resnet18_flips_90_resample_avp_016.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 4.96 ms
+Model loaded: 14.22 ms
+Prediction time for one image: 16.91 ms
+Compete inference time for one image: 36.12 ms
+DEVICE: cuda
+cuda is available
+Inference time masurement using cuda
+Image loaded: 1.38 ms
+Model loaded: 27.03 ms
+Prediction time for one image: 1.56 ms
+Compete inference time for one image: 30.11 ms
+GPU memory usage: 43.275390625 MB
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- 
 -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- 
\ No newline at end of file

train_results.txt

@@ -241,4 +241,20 @@ transforms.ToTensor()
         test accuracy: 95.13%
         balanced accuracy: 95.60%
         size: 43
-    checkpoints: /net/travail/bformanek/checkpoints/transfer_checkpoints_resnet18_flips_360
\ No newline at end of file
+    checkpoints: /net/travail/bformanek/checkpoints/transfer_checkpoints_resnet18_flips_360
+
+augmentation using horizontal, vertical flips and 90 degrees random rotation and resample until half resolution: 
+MODEL_NAME = 'resnet18_flips_90_resample'
+transforms.v2.RandomHorizontalFlip(p=0.5),
+transforms.v2.RandomVerticalFlip(p=0.5),
+transforms.v2.RandomRotation(degrees=90, expand=True),
+RandomResample(scale_factor=2),
+transforms.ToTensor()
+    time: 20 min 29 sec
+    Best epoch: 16
+        validation accuracy: 85.65%
+        validation loss: 2.1286
+        test accuracy: 91.59%
+        balanced accuracy: 88.54%
+        size: 43
+    checkpoints: /net/travail/bformanek/checkpoints/transfer_checkpoints_resnet18_flips_90_resample
\ No newline at end of file