From d4f39516693ad102e46fc38b90cf74d522dca86b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Formanek=20Bal=C3=A1zs=20Istv=C3=A1n?=
<formanek.balazs.istvan@itk.ppke.hu>
Date: Sun, 10 Nov 2024 17:06:03 +0100
Subject: [PATCH] time measurement locally, copy models and images - cuda not
available
---
.gitignore | 3 +
copy_best_models_to_local.py | 32 +++
cuda_test.py | 6 +
local_cpu_time_measurement_log.txt | 352 +++++++++++++++++++++++++++++
time_measure.py | 65 ++++--
5 files changed, 438 insertions(+), 20 deletions(-)
create mode 100644 copy_best_models_to_local.py
create mode 100644 cuda_test.py
create mode 100644 local_cpu_time_measurement_log.txt
diff --git a/.gitignore b/.gitignore
index 58f6d47..7a6650c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,5 @@
# ignore folders
__pycache__/
+
+local_models/
+local_images/
\ No newline at end of file
diff --git a/copy_best_models_to_local.py b/copy_best_models_to_local.py
new file mode 100644
index 0000000..567082c
--- /dev/null
+++ b/copy_best_models_to_local.py
@@ -0,0 +1,32 @@
+
+import shutil
+import os
+
+model_path_root = '/net/travail/bformanek/checkpoints/'
+local_root = '/net/cremi/bformanek/TRDP_II/local_models/'
+model_folders = ['transfer_checkpoints_efficientnet_b0_adam_amp_criterion_balanced',
+ 'transfer_checkpoints_resnet50_adam_amp_criterion_balanced',
+ 'transfer_checkpoints_resnet18_adam_amp_criterion_balanced',
+ 'transfer_checkpoints_resnet18.a2_in1k',
+ 'transfer_checkpoints_resnet18.fb_ssl_yfcc100m_ft_in1k',
+ 'transfer_checkpoints_mobilenetv3_small_100.lamb_in1k',
+ 'transfer_checkpoints_mobilenetv4_conv_large.e500_r256_in1k',
+ 'transfer_checkpoints_mobilenetv4_hybrid_medium.e500_r224_in1k',
+ 'transfer_checkpoints_resnet18_augmentation1',
+ 'transfer_checkpoints_resnet18_augmentation2',
+ 'transfer_checkpoints_resnet18_augmentation3',
+ 'transfer_checkpoints_resnet18_augmentation4']
+best_epochs = [26, 18, 25, 12, 22, 28, 4, 7, 21, 19, 3, 24]
+
+for i in range(len(model_folders)):
+ model_path = model_path_root + model_folders[i] + '/'
+ epoch_number = best_epochs[i]
+ model_name = f'avp_{epoch_number:03d}.pkl'
+ model_path = model_path + model_name
+
+ local_path = local_root + model_folders[i] + '_' + model_name
+
+ if not os.path.isfile(local_path):
+ print("Copy model " + model_path + " to " + local_path)
+ shutil.copyfile(model_path, local_path)
+
\ No newline at end of file
diff --git a/cuda_test.py b/cuda_test.py
new file mode 100644
index 0000000..2f7559e
--- /dev/null
+++ b/cuda_test.py
@@ -0,0 +1,6 @@
+import torch
+
+if torch.cuda.is_available():
+ print("available")
+else:
+ print("not available")
\ No newline at end of file
diff --git a/local_cpu_time_measurement_log.txt b/local_cpu_time_measurement_log.txt
new file mode 100644
index 0000000..e8687c3
--- /dev/null
+++ b/local_cpu_time_measurement_log.txt
@@ -0,0 +1,352 @@
+File "/autofs/unityaccount/cremi/bformanek/TRDP_II/time_measure.py", line 68, in inference_time_measurement
+ model = torch.load(model_path, map_location=torch.device('cpu'), weights_only=True)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/usr/lib/python3/dist-packages/torch/serialization.py", line 788, in load
+ raise pickle.UnpicklingError(UNSAFE_MESSAGE + str(e)) from None
+_pickle.UnpicklingError: Weights only load failed. Re-running `torch.load` with `weights_only` set to `False` will likely succeed, but it can result in arbitrary code execution.Do it only if you get the file from a trusted source. WeightsUnpickler error: Unsupported class timm.models.mobilenetv3.MobileNetV3
+
+cuda is not available
+
+(.venv) bformanek@falcon:~/TRDP_II$ python time_measure.py
+MEASUREMENT - 1
+Selected image path: /net/travail/bformanek/MRI_dataset/test/T2w/IXI222-Guys-0819-T2_358.1985_1_121.png
+Copied to local image path: /net/cremi/bformanek/TRDP_II/local_images/IXI222-Guys-0819-T2_358.1985_1_121.png
+MODEL:
+transfer_checkpoints_mobilenetv3_small_100.lamb_in1k_avp_028.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 4.49 ms
+Model loaded: 3728.52 ms
+Prediction time for one image: 398.29 ms
+Compete inference time for one image: 4131.33 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_efficientnet_b0_adam_amp_criterion_balanced_avp_026.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 2.10 ms
+Model loaded: 33.05 ms
+Prediction time for one image: 642.03 ms
+Compete inference time for one image: 677.21 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18.fb_ssl_yfcc100m_ft_in1k_avp_022.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 53.70 ms
+Model loaded: 19.95 ms
+Prediction time for one image: 17.19 ms
+Compete inference time for one image: 90.88 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_mobilenetv4_conv_large.e500_r256_in1k_avp_004.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.74 ms
+Model loaded: 62.56 ms
+Prediction time for one image: 53.74 ms
+Compete inference time for one image: 118.08 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_adam_amp_criterion_balanced_avp_025.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 77.87 ms
+Model loaded: 18.53 ms
+Prediction time for one image: 34.57 ms
+Compete inference time for one image: 131.02 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation2_avp_019.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 19.00 ms
+Model loaded: 24.10 ms
+Prediction time for one image: 9.96 ms
+Compete inference time for one image: 53.09 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation4_avp_024.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.58 ms
+Model loaded: 8.96 ms
+Prediction time for one image: 9.80 ms
+Compete inference time for one image: 20.36 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18.a2_in1k_avp_012.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.95 ms
+Model loaded: 9.13 ms
+Prediction time for one image: 9.67 ms
+Compete inference time for one image: 20.77 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet50_adam_amp_criterion_balanced_avp_018.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 8.71 ms
+Model loaded: 30.06 ms
+Prediction time for one image: 25.88 ms
+Compete inference time for one image: 64.68 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation1_avp_021.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 15.32 ms
+Model loaded: 20.67 ms
+Prediction time for one image: 9.16 ms
+Compete inference time for one image: 45.18 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation3_avp_003.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 18.13 ms
+Model loaded: 17.76 ms
+Prediction time for one image: 11.14 ms
+Compete inference time for one image: 47.07 ms
+DEVICE: cuda
+cuda is not available
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+MEASUREMENT - 2
+Selected image path: /net/travail/bformanek/MRI_dataset/test/T1w/NACC922027_2_T1w_609.3745_1_142.png
+Copied to local image path: /net/cremi/bformanek/TRDP_II/local_images/NACC922027_2_T1w_609.3745_1_142.png
+MODEL:
+transfer_checkpoints_mobilenetv3_small_100.lamb_in1k_avp_028.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 12.95 ms
+Model loaded: 20.07 ms
+Prediction time for one image: 1151.98 ms
+Compete inference time for one image: 1185.02 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_efficientnet_b0_adam_amp_criterion_balanced_avp_026.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 38.82 ms
+Model loaded: 26.92 ms
+Prediction time for one image: 538.51 ms
+Compete inference time for one image: 604.30 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18.fb_ssl_yfcc100m_ft_in1k_avp_022.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 19.23 ms
+Model loaded: 12.14 ms
+Prediction time for one image: 95.33 ms
+Compete inference time for one image: 132.12 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_mobilenetv4_conv_large.e500_r256_in1k_avp_004.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 2.73 ms
+Model loaded: 46.44 ms
+Prediction time for one image: 35.26 ms
+Compete inference time for one image: 84.47 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_adam_amp_criterion_balanced_avp_025.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 36.70 ms
+Model loaded: 18.62 ms
+Prediction time for one image: 86.22 ms
+Compete inference time for one image: 141.67 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation2_avp_019.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 2.70 ms
+Model loaded: 19.06 ms
+Prediction time for one image: 11.38 ms
+Compete inference time for one image: 33.17 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation4_avp_024.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 2.57 ms
+Model loaded: 18.53 ms
+Prediction time for one image: 11.01 ms
+Compete inference time for one image: 32.15 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18.a2_in1k_avp_012.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 2.46 ms
+Model loaded: 38.58 ms
+Prediction time for one image: 15.69 ms
+Compete inference time for one image: 56.76 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet50_adam_amp_criterion_balanced_avp_018.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.95 ms
+Model loaded: 18.63 ms
+Prediction time for one image: 22.37 ms
+Compete inference time for one image: 42.98 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation1_avp_021.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 2.11 ms
+Model loaded: 23.16 ms
+Prediction time for one image: 9.36 ms
+Compete inference time for one image: 34.65 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation3_avp_003.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.98 ms
+Model loaded: 21.85 ms
+Prediction time for one image: 9.68 ms
+Compete inference time for one image: 33.53 ms
+DEVICE: cuda
+cuda is not available
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+MEASUREMENT - 3
+Selected image path: /net/travail/bformanek/MRI_dataset/test/FLAIR/NACC361368_1_FLAIR_149.6907_0_112.png
+Copied to local image path: /net/cremi/bformanek/TRDP_II/local_images/NACC361368_1_FLAIR_149.6907_0_112.png
+MODEL:
+transfer_checkpoints_mobilenetv3_small_100.lamb_in1k_avp_028.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.80 ms
+Model loaded: 6.07 ms
+Prediction time for one image: 1277.93 ms
+Compete inference time for one image: 1285.82 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_efficientnet_b0_adam_amp_criterion_balanced_avp_026.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 40.67 ms
+Model loaded: 19.43 ms
+Prediction time for one image: 1256.69 ms
+Compete inference time for one image: 1316.82 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18.fb_ssl_yfcc100m_ft_in1k_avp_022.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 60.37 ms
+Model loaded: 9.24 ms
+Prediction time for one image: 10.24 ms
+Compete inference time for one image: 79.88 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_mobilenetv4_conv_large.e500_r256_in1k_avp_004.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 9.21 ms
+Model loaded: 173.25 ms
+Prediction time for one image: 26.67 ms
+Compete inference time for one image: 209.17 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_adam_amp_criterion_balanced_avp_025.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 42.93 ms
+Model loaded: 18.17 ms
+Prediction time for one image: 97.93 ms
+Compete inference time for one image: 159.07 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation2_avp_019.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.96 ms
+Model loaded: 31.92 ms
+Prediction time for one image: 15.63 ms
+Compete inference time for one image: 49.54 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation4_avp_024.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.63 ms
+Model loaded: 8.51 ms
+Prediction time for one image: 9.85 ms
+Compete inference time for one image: 20.02 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18.a2_in1k_avp_012.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.78 ms
+Model loaded: 17.40 ms
+Prediction time for one image: 22.67 ms
+Compete inference time for one image: 41.88 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet50_adam_amp_criterion_balanced_avp_018.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.53 ms
+Model loaded: 181.73 ms
+Prediction time for one image: 21.06 ms
+Compete inference time for one image: 204.35 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation1_avp_021.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 1.72 ms
+Model loaded: 7.92 ms
+Prediction time for one image: 8.81 ms
+Compete inference time for one image: 18.46 ms
+DEVICE: cuda
+cuda is not available
+MODEL:
+transfer_checkpoints_resnet18_augmentation3_avp_003.pkl
+DEVICE: cpu
+Inference time masurement using cpu
+Image loaded: 239.88 ms
+Model loaded: 7.49 ms
+Prediction time for one image: 8.06 ms
+Compete inference time for one image: 255.45 ms
+DEVICE: cuda
+cuda is not available
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
\ No newline at end of file
diff --git a/time_measure.py b/time_measure.py
index 7750577..66a26c0 100644
--- a/time_measure.py
+++ b/time_measure.py
@@ -6,6 +6,7 @@ import time
import random
import os
from PIL import Image
+import shutil
# Get a random image path from given folder
def get_random_image_path(folder):
@@ -20,6 +21,18 @@ def get_random_image_path(folder):
random_image_path = random.choice(all_images)
return random_image_path
+def copy_image_to_local(remote_image_folder, local_images_folder):
+ # Randomly select an image path, in other case it might be given by the user
+ image_path = get_random_image_path(remote_image_folder)
+ print(f"Selected image path: {image_path}")
+
+ # copy image to local
+ local_image_path = local_images_folder + os.path.basename(image_path)
+ shutil.copyfile(image_path, local_image_path)
+ print(f"Copied to local image path: {local_image_path}")
+
+ return local_image_path
+
def inference_time_measurement(model_path, image_path, transform, device = 'cuda', num_wokres = 8):
# check cuda
if device == 'cuda':
@@ -47,22 +60,19 @@ def inference_time_measurement(model_path, image_path, transform, device = 'cuda
load_image_time = (load_image_time_end - load_image_time_start) * 1000
print(f'Image loaded: {load_image_time:.2f} ms')
-
# 2) Measure Model Loading Time
# Get the model name
# print("Loading model")
load_model_time_start = time.time() # start timer
model = torch.load(model_path, map_location=torch.device('cpu'), weights_only=False)
+ model.eval() # Set model to evaluation mode
model.to(device)
load_model_time_end = time.time()
load_model_time = (load_model_time_end - load_model_time_start) * 1000 # elapsed time in milliseconds
print(f'Model loaded: {load_model_time:.2f} ms')
- # Set model to evaluation mode
- model.eval()
-
# 3) Predict on the single image
image = image.to(device)
prediction_time_start = time.time()
@@ -85,26 +95,41 @@ def inference_time_measurement(model_path, image_path, transform, device = 'cuda
# Main function
+number_of_measurements = 3
-# Randomly select an image path, in other case it might be given by the user
-TEST_FOLDER = '/net/travail/bformanek/MRI_dataset/test'
-image_path = get_random_image_path(TEST_FOLDER)
-print(f"Selected image path: {image_path}")
-
-# inputs
-model_path = '/net/travail/bformanek/checkpoints/transfer_checkpoints_resnet18_augmentation4/'
-epoch_number = 24
-model_name = f'avp_{epoch_number:03d}.pkl'
-model_path = model_path + model_name
-device = 'cuda' # 'cpu' 'cuda'
# Define the transformation for the test data
transform = transforms.Compose([
transforms.Resize(224),
transforms.ToTensor()
])
-inference_time_measurement(model_path = model_path,
- image_path = image_path,
- transform = transform,
- device = device,
- num_wokres = 8)
+test_image_folder = '/net/travail/bformanek/MRI_dataset/test'
+local_images_folder = '/net/cremi/bformanek/TRDP_II/local_images/'
+local_model_folder = '/net/cremi/bformanek/TRDP_II/local_models/'
+
+for i in range(number_of_measurements):
+ print(f"MEASUREMENT - {i + 1}" )
+ # get image to local folder
+ local_image_path = copy_image_to_local(test_image_folder, local_images_folder)
+ for model_name in os.listdir(local_model_folder) :
+ print("MODEL: ")
+ print(model_name)
+
+ for device in ['cpu', 'cuda']:
+ print("DEVICE: " + device)
+
+ if device == 'cuda':
+ if not torch.cuda.is_available():
+ print('cuda is not available')
+ break
+
+ model_path = local_model_folder + model_name
+
+ inference_time_measurement(model_path = model_path,
+ image_path = local_image_path,
+ transform = transform,
+ device = device,
+ num_wokres = 8)
+ print("-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- ")
+ print("-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- ")
+ os.remove(local_image_path)
\ No newline at end of file
--
GitLab